You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by im...@apache.org on 2025/04/29 20:09:47 UTC
(sedona) branch SEDONA-725-restructure-spark-python-package created (now db7af60eae)
This is an automated email from the ASF dual-hosted git repository.
imbruced pushed a change to branch SEDONA-725-restructure-spark-python-package
in repository https://gitbox.apache.org/repos/asf/sedona.git
at db7af60eae SEDONA-725 rearrange the spark module
This branch includes the following new commits:
new 36f8e2945c SEDONA-725 Add pyflink to Sedona.
new db7af60eae SEDONA-725 rearrange the spark module
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(sedona) 01/02: SEDONA-725 Add pyflink to Sedona.
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imbruced pushed a commit to branch SEDONA-725-restructure-spark-python-package
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 36f8e2945c47708ad8de9baa69e2d02957f7569d
Author: pawelkocinski <pa...@gmail.com>
AuthorDate: Sun Mar 23 19:06:02 2025 +0100
SEDONA-725 Add pyflink to Sedona.
---
.github/workflows/pyflink.yml | 78 ++++
docs/api/sql/Raster-visualizer.md | 2 +-
docs/api/sql/Visualization_SedonaKepler.md | 2 +-
docs/api/sql/Visualization_SedonaPyDeck.md | 2 +-
docs/setup/install-python.md | 4 +-
docs/tutorial/concepts/clustering-algorithms.md | 2 +-
docs/tutorial/files/stac-sedona-spark.md | 2 +-
docs/tutorial/geopandas-shapely.md | 2 +-
docs/tutorial/rdd.md | 48 +--
docs/tutorial/sql.md | 30 +-
docs/usecases/ApacheSedonaCore.ipynb | 73 ++--
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 2 +-
.../usecases/contrib/ApacheSedonaImageFilter.ipynb | 9 +-
.../contrib/NdviSentinelApacheSedona.ipynb | 33 +-
.../contrib/VectorAnalisisApacheSedona.ipynb | 31 +-
docs/usecases/contrib/foot-traffic.ipynb | 12 +-
python/pytest.ini | 3 +
python/sedona/flink/__init__.py | 14 +
python/sedona/flink/context.py | 35 ++
python/sedona/raster_utils/__init__.py | 16 -
python/sedona/spark/SedonaContext.py | 4 +-
python/sedona/spark/__init__.py | 44 ++-
.../sedona/{ => spark}/core/SpatialRDD/__init__.py | 0
.../{ => spark}/core/SpatialRDD/circle_rdd.py | 14 +-
.../{ => spark}/core/SpatialRDD/linestring_rdd.py | 16 +-
.../{ => spark}/core/SpatialRDD/point_rdd.py | 12 +-
.../{ => spark}/core/SpatialRDD/polygon_rdd.py | 14 +-
.../{ => spark}/core/SpatialRDD/rectangle_rdd.py | 11 +-
.../{ => spark}/core/SpatialRDD/spatial_rdd.py | 26 +-
.../core/SpatialRDD/spatial_rdd_factory.py | 2 +-
python/sedona/{utils => spark/core}/__init__.py | 5 +-
python/sedona/{ => spark}/core/enums/__init__.py | 0
.../{ => spark}/core/enums/file_data_splitter.py | 4 +-
python/sedona/{ => spark}/core/enums/grid_type.py | 4 +-
python/sedona/{ => spark}/core/enums/index_type.py | 4 +-
.../{ => spark}/core/enums/join_build_side.py | 3 +-
python/sedona/{ => spark}/core/enums/spatial.py | 0
.../{ => spark}/core/formatMapper/__init__.py | 0
.../{ => spark}/core/formatMapper/disc_utils.py | 6 +-
.../core/formatMapper/geo_json_reader.py | 6 +-
.../{ => spark}/core/formatMapper/geo_reader.py | 2 +-
.../core/formatMapper/shapefileParser/__init__.py | 0
.../shapefileParser/shape_file_reader.py | 8 +-
.../{ => spark}/core/formatMapper/wkb_reader.py | 6 +-
.../{ => spark}/core/formatMapper/wkt_reader.py | 6 +-
.../sedona/{core => spark/core/geom}/__init__.py | 0
python/sedona/{ => spark}/core/geom/circle.py | 0
python/sedona/{ => spark}/core/geom/envelope.py | 0
python/sedona/{ => spark}/core/geom/geography.py | 0
.../geom => spark/core/geom/shapely1}/__init__.py | 0
.../{ => spark}/core/geom/shapely1/circle.py | 2 +-
.../{ => spark}/core/geom/shapely1/envelope.py | 5 +-
.../core/geom/shapely2}/__init__.py | 0
.../{ => spark}/core/geom/shapely2/circle.py | 3 +-
.../{ => spark}/core/geom/shapely2/envelope.py | 4 +-
.../geom/shapely2 => spark/core/jvm}/__init__.py | 0
python/sedona/{ => spark}/core/jvm/abstract.py | 0
python/sedona/{ => spark}/core/jvm/config.py | 2 +-
python/sedona/{ => spark}/core/jvm/translate.py | 0
.../{ => spark}/core/spatialOperator/__init__.py | 0
.../core/spatialOperator/join_params.py | 6 +-
.../{ => spark}/core/spatialOperator/join_query.py | 8 +-
.../core/spatialOperator/join_query_raw.py | 8 +-
.../{ => spark}/core/spatialOperator/knn_query.py | 12 +-
.../core/spatialOperator/range_query.py | 6 +-
.../core/spatialOperator/range_query_raw.py | 8 +-
.../sedona/{ => spark}/core/spatialOperator/rdd.py | 4 +-
python/sedona/{ => spark}/core/utils.py | 0
python/sedona/spark/geoarrow/__init__.py | 3 +
.../sedona/{utils => spark/geoarrow}/geoarrow.py | 4 +-
python/sedona/{ => spark}/maps/SedonaKepler.py | 4 +-
python/sedona/{ => spark}/maps/SedonaMapUtils.py | 4 +-
python/sedona/{ => spark}/maps/SedonaPyDeck.py | 2 +-
python/sedona/{core/jvm => spark/maps}/__init__.py | 0
python/sedona/{maps => spark/raster}/__init__.py | 0
python/sedona/{ => spark}/raster/awt_raster.py | 0
python/sedona/{ => spark}/raster/data_buffer.py | 0
python/sedona/{ => spark}/raster/meta.py | 0
python/sedona/{ => spark}/raster/raster_serde.py | 0
python/sedona/{ => spark}/raster/sample_model.py | 0
python/sedona/{ => spark}/raster/sedona_raster.py | 0
.../sedona/{ => spark}/raster_utils/SedonaUtils.py | 2 +-
.../{raster => spark/raster_utils}/__init__.py | 0
python/sedona/{ => spark}/register/__init__.py | 2 +-
.../sedona/{ => spark}/register/geo_registrator.py | 8 +-
python/sedona/{ => spark}/register/java_libs.py | 0
python/sedona/{ => spark}/sql/__init__.py | 21 +-
python/sedona/{ => spark}/sql/connect.py | 0
python/sedona/{ => spark}/sql/dataframe_api.py | 2 +-
python/sedona/{ => spark}/sql/exceptions.py | 0
python/sedona/{ => spark}/sql/functions.py | 4 +-
python/sedona/{ => spark}/sql/st_aggregates.py | 2 +-
python/sedona/{ => spark}/sql/st_constructors.py | 2 +-
python/sedona/{ => spark}/sql/st_functions.py | 2 +-
python/sedona/{ => spark}/sql/st_predicates.py | 2 +-
python/sedona/{ => spark}/sql/types.py | 14 +-
.../{core/geom/shapely1 => spark/stac}/__init__.py | 1 +
python/sedona/{ => spark}/stac/client.py | 2 +-
.../sedona/{ => spark}/stac/collection_client.py | 0
.../tools.py => sedona/spark/stats/__init__.py} | 21 +-
.../{ => spark}/stats/clustering/__init__.py | 0
.../sedona/{ => spark}/stats/clustering/dbscan.py | 0
.../stats/hotspot_detection/__init__.py | 0
.../stats/hotspot_detection/getis_ord.py | 0
.../stats/outlier_detection/__init__.py | 0
.../outlier_detection/local_outlier_factor.py | 0
python/sedona/{ => spark}/stats/weighting.py | 0
python/sedona/{ => spark}/utils/__init__.py | 0
python/sedona/{ => spark}/utils/abstract_parser.py | 0
python/sedona/{ => spark}/utils/adapter.py | 10 +-
python/sedona/{ => spark}/utils/binary_parser.py | 0
python/sedona/{ => spark}/utils/decorators.py | 2 +-
.../sedona/{ => spark}/utils/geometry_adapter.py | 6 +-
python/sedona/{ => spark}/utils/geometry_serde.py | 0
.../{ => spark}/utils/geometry_serde_general.py | 0
python/sedona/{ => spark}/utils/jvm.py | 4 +-
python/sedona/{ => spark}/utils/meta.py | 0
python/sedona/{ => spark}/utils/prep.py | 6 +-
python/sedona/{ => spark}/utils/serde.py | 2 +-
.../sedona/{ => spark}/utils/spatial_rdd_parser.py | 11 +-
.../sedona/{ => spark}/utils/structured_adapter.py | 4 +-
python/sedona/{ => spark}/utils/types.py | 0
python/sedona/stac/__init__.py | 29 +-
python/sedona/stats/__init__.py | 16 -
python/sedona/stats/clustering/__init__.py | 21 --
python/sedona/stats/clustering/dbscan/__init__.py | 11 +
python/sedona/stats/hotspot_detection/__init__.py | 18 -
.../stats/hotspot_detection/getis_ord/__init__.py | 9 +
python/sedona/stats/outlier_detection/__init__.py | 18 -
.../local_outlier_factor/__init__.py | 13 +
python/sedona/stats/weighting/__init__.py | 9 +
python/sedona/utils/__init__.py | 20 -
python/sedona/utils/geoarrow/__init__.py | 11 +
python/sedona/utils/structured_adapter/__init__.py | 13 +
python/setup.py | 3 +-
python/tests/__init__.py | 7 +-
.../core/test_avoiding_python_jvm_serde_df.py | 21 +-
.../core/test_avoiding_python_jvm_serde_to_rdd.py | 17 +-
python/tests/core/test_config.py | 2 +-
python/tests/core/test_core_geom_primitives.py | 2 +-
python/tests/core/test_core_rdd.py | 6 +-
python/tests/core/test_rdd.py | 17 +-
python/tests/core/test_spatial_rdd_from_disc.py | 11 +-
python/tests/flink/conftest.py | 43 +++
python/tests/flink/test_flink_registration.py | 45 +++
python/tests/format_mapper/test_geo_json_reader.py | 7 +-
.../tests/format_mapper/test_shapefile_reader.py | 10 +-
python/tests/format_mapper/test_wkb_reader.py | 4 +-
python/tests/format_mapper/test_wkt_reader.py | 4 +-
.../tests/maps/test_sedonakepler_visualization.py | 2 +-
python/tests/maps/test_sedonapydeck.py | 2 +-
python/tests/properties/crs_transform.py | 6 +-
python/tests/properties/linestring_properties.py | 6 +-
python/tests/properties/point_properties.py | 7 +-
python/tests/properties/polygon_properties.py | 7 +-
python/tests/raster/test_meta.py | 3 +-
python/tests/raster/test_pandas_udf.py | 5 +-
python/tests/raster/test_serde.py | 2 +-
python/tests/raster_viz_utils/test_sedonautils.py | 3 +-
.../serialization/test_direct_serialization.py | 2 +-
.../tests/serialization/test_rdd_serialization.py | 6 +-
python/tests/serialization/test_serializers.py | 2 +-
.../tests/serialization/test_with_sc_parellize.py | 5 +-
python/tests/spatial_operator/test_join_base.py | 12 +-
.../test_join_query_correctness.py | 11 +-
.../tests/spatial_operator/test_linestring_join.py | 11 +-
.../tests/spatial_operator/test_linestring_knn.py | 9 +-
.../spatial_operator/test_linestring_range.py | 10 +-
python/tests/spatial_operator/test_point_join.py | 11 +-
python/tests/spatial_operator/test_point_knn.py | 10 +-
python/tests/spatial_operator/test_point_range.py | 10 +-
python/tests/spatial_operator/test_polygon_join.py | 11 +-
python/tests/spatial_operator/test_polygon_knn.py | 10 +-
.../tests/spatial_operator/test_polygon_range.py | 10 +-
.../tests/spatial_operator/test_rectangle_join.py | 12 +-
.../tests/spatial_operator/test_rectangle_knn.py | 12 +-
.../tests/spatial_operator/test_rectangle_range.py | 10 +-
python/tests/spatial_rdd/test_circle_rdd.py | 2 +-
python/tests/spatial_rdd/test_linestring_rdd.py | 8 +-
python/tests/spatial_rdd/test_point_rdd.py | 10 +-
python/tests/spatial_rdd/test_polygon_rdd.py | 9 +-
python/tests/spatial_rdd/test_rectangle_rdd.py | 10 +-
python/tests/spatial_rdd/test_spatial_rdd.py | 14 +-
.../tests/spatial_rdd/test_spatial_rdd_writer.py | 9 +-
python/tests/sql/resource/sample_data.py | 2 +-
python/tests/sql/test_adapter.py | 17 +-
python/tests/sql/test_dataframe_api.py | 16 +-
python/tests/sql/test_function.py | 2 +-
python/tests/sql/test_geography.py | 5 +-
python/tests/sql/test_shapefile.py | 3 +-
.../sql/test_spatial_rdd_to_spatial_dataframe.py | 9 +-
python/tests/sql/test_st_function_imports.py | 7 +-
python/tests/sql/test_structured_adapter.py | 8 +-
python/tests/stac/test_client.py | 4 +-
python/tests/stac/test_collection_client.py | 4 +-
python/tests/stats/test_dbscan.py | 6 +-
python/tests/stats/test_getis_ord.py | 7 +-
python/tests/stats/test_local_outlier_factor.py | 6 +-
python/tests/stats/test_weighting.py | 4 +-
.../streaming/spark/test_constructor_functions.py | 2 +-
python/tests/test_assign_raw_spatial_rdd.py | 2 +-
python/tests/test_base.py | 2 +-
python/tests/test_circle.py | 4 +-
python/tests/test_multiple_meta.py | 2 +-
python/tests/test_scala_example.py | 420 +++++++++++----------
python/tests/tools.py | 10 +-
.../test_arrow_conversion_geopandas_to_sedona.py | 4 +-
python/tests/utils/test_crs_transformation.py | 16 +-
python/tests/utils/test_geo_spark_meta.py | 2 +-
python/tests/utils/test_geoarrow.py | 2 +-
python/tests/utils/test_geometry_serde.py | 2 +-
python/tests/utils/test_geomserde_speedup.py | 2 +-
python/tests/utils/test_pandas_arrow_udf.py | 4 +-
.../spark/sql/sedona_sql/UDT/GeographyUDT.scala | 2 +-
.../spark/sql/sedona_sql/UDT/GeometryUDT.scala | 2 +-
.../spark/sql/sedona_sql/UDT/RasterUDT.scala | 2 +-
216 files changed, 1090 insertions(+), 958 deletions(-)
diff --git a/.github/workflows/pyflink.yml b/.github/workflows/pyflink.yml
new file mode 100644
index 0000000000..7e9732190d
--- /dev/null
+++ b/.github/workflows/pyflink.yml
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Sedona Pyflink Test
+
+on:
+ push:
+ branches:
+ - master
+ paths:
+ - 'common/**'
+ - 'flink/**'
+ - 'flink-shaded/**'
+ - 'pom.xml'
+ - 'python/**'
+ - '.github/workflows/pyflink.yml'
+ pull_request:
+ branches:
+ - '*'
+ paths:
+ - 'common/**'
+ - 'flink/**'
+ - 'flink-shaded/**'
+ - 'pom.xml'
+ - 'python/**'
+ - '.github/workflows/pyflink.yml'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test:
+ runs-on: ubuntu-22.04
+ strategy:
+ matrix:
+ include:
+ - python: '3.10'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
+ with:
+ distribution: 'zulu'
+ java-version: '8'
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ - run: sudo apt-get -y install python3-pip python-dev-is-python3
+ - run: mvn package -pl "org.apache.sedona:sedona-flink-shaded_2.12" -am -DskipTests
+ - run: sudo pip3 install -U setuptools
+ - run: sudo pip3 install -U wheel
+ - run: sudo pip3 install -U virtualenvwrapper
+ - run: python3 -m pip install uv
+ - run: cd python
+ - run: rm pyproject.toml
+ - run: uv init --no-workspace
+ - run: uv add apache-flink==1.20.1 shapely attr setuptools
+ - run: uv add pytest --dev
+ - run: |
+ wget https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/1.7.1-28.5/geotools-wrapper-1.7.1-28.5.jar
+ export SEDONA_PYFLINK_EXTRA_JARS=${PWD}/$(find flink-shaded/target -name sedona-flink*.jar),${PWD}/geotools-wrapper-1.7.1-28.5.jar
+ pwd
+ ls -l
+ (cd python; PYTHONPATH=$(pwd) uv run pytest -v -m flink -s ./tests/flink)
diff --git a/docs/api/sql/Raster-visualizer.md b/docs/api/sql/Raster-visualizer.md
index f0b83cb165..ce58eb986c 100644
--- a/docs/api/sql/Raster-visualizer.md
+++ b/docs/api/sql/Raster-visualizer.md
@@ -77,7 +77,7 @@ Output:
Example:
```python
-from sedona.raster_utils.SedonaUtils import SedonaUtils
+from sedona.spark import SedonaUtils
# Or from sedona.spark import *
diff --git a/docs/api/sql/Visualization_SedonaKepler.md b/docs/api/sql/Visualization_SedonaKepler.md
index 4889109529..fa4fb4b302 100644
--- a/docs/api/sql/Visualization_SedonaKepler.md
+++ b/docs/api/sql/Visualization_SedonaKepler.md
@@ -28,7 +28,7 @@ from sedona.spark import *
Alternatively it can also be imported using:
```python
-from sedona.maps.SedonaKepler import SedonaKepler
+from sedona.spark import SedonaKepler
```
Following are details on all the APIs exposed via SedonaKepler:
diff --git a/docs/api/sql/Visualization_SedonaPyDeck.md b/docs/api/sql/Visualization_SedonaPyDeck.md
index a9b0f5a1c8..62a4f061a2 100644
--- a/docs/api/sql/Visualization_SedonaPyDeck.md
+++ b/docs/api/sql/Visualization_SedonaPyDeck.md
@@ -28,7 +28,7 @@ from sedona.spark import *
Alternatively it can also be imported using:
```python
-from sedona.maps.SedonaPyDeck import SedonaPyDeck
+from sedona.spark import SedonaPyDeck
```
!!!Note
diff --git a/docs/setup/install-python.md b/docs/setup/install-python.md
index 0ea1e2cc8a..02cb51d475 100644
--- a/docs/setup/install-python.md
+++ b/docs/setup/install-python.md
@@ -85,8 +85,8 @@ SedonaRegistrator is deprecated in Sedona 1.4.1 and later versions. Please use t
```python
from pyspark.sql import SparkSession
-from sedona.register import SedonaRegistrator
-from sedona.utils import SedonaKryoRegistrator, KryoSerializer
+from sedona.spark import SedonaRegistrator
+from sedona.spark import SedonaKryoRegistrator, KryoSerializer
spark = (
SparkSession.builder.appName("appName")
diff --git a/docs/tutorial/concepts/clustering-algorithms.md b/docs/tutorial/concepts/clustering-algorithms.md
index 79cb72cd8f..460e252ad6 100644
--- a/docs/tutorial/concepts/clustering-algorithms.md
+++ b/docs/tutorial/concepts/clustering-algorithms.md
@@ -96,7 +96,7 @@ Here are the contents of the DataFrame:
Here’s how to run the DBSCAN algorithm:
```python
-from sedona.stats.clustering.dbscan import dbscan
+from sedona.spark.stats import dbscan
dbscan(df, 1.0, 3).orderBy("id").show()
```
diff --git a/docs/tutorial/files/stac-sedona-spark.md b/docs/tutorial/files/stac-sedona-spark.md
index 864b6ef9d6..78b8616f83 100644
--- a/docs/tutorial/files/stac-sedona-spark.md
+++ b/docs/tutorial/files/stac-sedona-spark.md
@@ -204,7 +204,7 @@ The Python API allows you to interact with a SpatioTemporal Asset Catalog (STAC)
#### Initialize the Client
```python
-from sedona.stac.client import Client
+from sedona.spark.stac import Client
# Initialize the client
client = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1")
diff --git a/docs/tutorial/geopandas-shapely.md b/docs/tutorial/geopandas-shapely.md
index c3fcbf2dc3..66c4c068f3 100644
--- a/docs/tutorial/geopandas-shapely.md
+++ b/docs/tutorial/geopandas-shapely.md
@@ -81,7 +81,7 @@ def create_spatial_dataframe(spark: SparkSession, gdf: gpd.GeoDataFrame) -> Data
Example:
```python
-from sedona.utils.geoarrow import create_spatial_dataframe
+from sedona.spark.geoarrow import create_spatial_dataframe
create_spatial_dataframe(spark, gdf)
```
diff --git a/docs/tutorial/rdd.md b/docs/tutorial/rdd.md
index d3b27896f7..23b94d519f 100644
--- a/docs/tutorial/rdd.md
+++ b/docs/tutorial/rdd.md
@@ -51,7 +51,7 @@ Please refer to [Create a Geometry type column](sql.md#create-a-geometry-type-co
=== "Python"
```python
- from sedona.utils.structured_adapter import StructuredAdapter
+ from sedona.spark import StructuredAdapter
spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty")
```
@@ -165,8 +165,8 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
=== "Python"
```python
- from sedona.core.geom.envelope import Envelope
- from sedona.core.spatialOperator import RangeQuery
+ from sedona.spark import Envelope
+ from sedona.spark import RangeQuery
range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
consider_boundary_intersection = False ## Only return gemeotries fully covered by the window
@@ -179,9 +179,9 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
Example:
```python
- from sedona.core.geom.envelope import Envelope
- from sedona.core.spatialOperator import RangeQueryRaw
- from sedona.utils.adapter import Adapter
+ from sedona.spark import Envelope
+ from sedona.spark import RangeQueryRaw
+ from sedona.spark import Adapter
range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
consider_boundary_intersection = False ## Only return gemeotries fully covered by the window
@@ -283,9 +283,9 @@ To utilize a spatial index in a spatial range query, use the following code:
=== "Python"
```python
- from sedona.core.geom.envelope import Envelope
- from sedona.core.enums import IndexType
- from sedona.core.spatialOperator import RangeQuery
+ from sedona.spark import Envelope
+ from sedona.spark import IndexType
+ from sedona.spark import RangeQuery
range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
consider_boundary_intersection = False ## Only return gemeotries fully covered by the window
@@ -379,7 +379,7 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
=== "Python"
```python
- from sedona.core.spatialOperator import KNNQuery
+ from sedona.spark import KNNQuery
from shapely.geometry import Point
point = Point(-84.01, 34.01)
@@ -446,8 +446,8 @@ To utilize a spatial index in a spatial KNN query, use the following code:
=== "Python"
```python
- from sedona.core.spatialOperator import KNNQuery
- from sedona.core.enums import IndexType
+ from sedona.spark import KNNQuery
+ from sedona.spark import IndexType
from shapely.geometry import Point
point = Point(-84.01, 34.01)
@@ -518,8 +518,8 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin
=== "Python"
```python
- from sedona.core.enums import GridType
- from sedona.core.spatialOperator import JoinQuery
+ from sedona.spark import GridType
+ from sedona.spark import JoinQuery
consider_boundary_intersection = False ## Only return geometries fully covered by each query window in queryWindowRDD
using_index = False
@@ -610,9 +610,9 @@ To utilize a spatial index in a spatial join query, use the following code:
=== "Python"
```python
- from sedona.core.enums import GridType
- from sedona.core.enums import IndexType
- from sedona.core.spatialOperator import JoinQuery
+ from sedona.spark import GridType
+ from sedona.spark import IndexType
+ from sedona.spark import JoinQuery
object_rdd.spatialPartitioning(GridType.KDBTREE)
query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
@@ -676,10 +676,10 @@ The index should be built on either one of two SpatialRDDs. In general, you shou
Example:
```python
- from sedona.core.SpatialRDD import CircleRDD
- from sedona.core.enums import GridType
- from sedona.core.spatialOperator import JoinQueryRaw
- from sedona.utils.structured_adapter import StructuredAdapter
+ from sedona.spark import CircleRDD
+ from sedona.spark import GridType
+ from sedona.spark import JoinQueryRaw
+ from sedona.spark import StructuredAdapter
object_rdd.analyze()
@@ -743,9 +743,9 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin
=== "Python"
```python
- from sedona.core.SpatialRDD import CircleRDD
- from sedona.core.enums import GridType
- from sedona.core.spatialOperator import JoinQuery
+ from sedona.spark import CircleRDD
+ from sedona.spark import GridType
+ from sedona.spark import JoinQuery
object_rdd.analyze()
diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md
index 71e8e7d8a1..3cd3b5d734 100644
--- a/docs/tutorial/sql.md
+++ b/docs/tutorial/sql.md
@@ -614,7 +614,7 @@ The first parameter is the dataframe, the next two are the epsilon and min_point
=== "Python"
```python
- from sedona.stats.clustering.dbscan import dbscan
+ from sedona.spark.stats import dbscan
dbscan(df, 0.1, 5).show()
```
@@ -670,7 +670,7 @@ The first parameter is the dataframe, the next is the number of nearest neighbor
=== "Python"
```python
- from sedona.stats.outlier_detection.local_outlier_factor import local_outlier_factor
+ from sedona.spark.stats import local_outlier_factor
local_outlier_factor(df, 20).show()
```
@@ -737,8 +737,8 @@ Using Gi involves first generating the neighbors list for each record, then call
=== "Python"
```python
- from sedona.stats.weighting import add_binary_distance_band_column
- from sedona.stats.hotspot_detection.getis_ord import g_local
+ from sedona.spark.stats import add_binary_distance_band_column
+ from sedona.spark.stats import g_local
distance_radius = 1.0
weighted_df = addBinaryDistanceBandColumn(df, distance_radius)
@@ -966,7 +966,7 @@ This UDF example takes a geometry type input and returns a primitive type output
=== "Python"
```python
- from sedona.sql.types import GeometryType
+ from sedona.spark.sql.types import GeometryType
from pyspark.sql.types import DoubleType
def lengthPoly(geom: GeometryType()):
@@ -1025,7 +1025,7 @@ This UDF example takes a geometry type input and returns a geometry type output:
=== "Python"
```python
- from sedona.sql.types import GeometryType
+ from sedona.spark import GeometryType
from pyspark.sql.types import DoubleType
def bufferFixed(geom: GeometryType()):
@@ -1083,7 +1083,7 @@ This UDF example takes a geometry type input and a primitive type input and retu
=== "Python"
```python
- from sedona.sql.types import GeometryType
+ from sedona.spark import GeometryType
from pyspark.sql.types import DoubleType
def bufferIt(geom: GeometryType(), distance: DoubleType()):
@@ -1165,7 +1165,7 @@ This UDF example takes a geometry type input and a primitive type input and retu
=== "Python"
```python
- from sedona.sql.types import GeometryType
+ from sedona.spark import GeometryType
from pyspark.sql.types import *
schemaUDF = StructType([
@@ -1230,7 +1230,7 @@ a given geometry.
```python
import shapely.geometry.base as b
-from sedona.sql.functions import sedona_vectorized_udf
+from sedona.spark import sedona_vectorized_udf
@sedona_vectorized_udf(return_type=GeometryType())
def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry:
@@ -1241,8 +1241,8 @@ def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry:
```python
import geopandas as gpd
-from sedona.sql.functions import sedona_vectorized_udf, SedonaUDFType
-from sedona.sql.types import GeometryType
+from sedona.spark import sedona_vectorized_udf, SedonaUDFType
+from sedona.spark import GeometryType
@sedona_vectorized_udf(udf_type=SedonaUDFType.GEO_SERIES, return_type=GeometryType())
@@ -1339,7 +1339,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD.
=== "Python"
```python
- from sedona.utils.structured_adapter import StructuredAdapter
+ from sedona.spark import StructuredAdapter
spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty")
```
@@ -1365,7 +1365,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD. Ple
=== "Python"
```python
- from sedona.utils.adapter import StructuredAdapter
+ from sedona.spark import StructuredAdapter
spatialDf = StructuredAdapter.toDf(spatialRDD, sedona)
```
@@ -1401,7 +1401,7 @@ You can use `StructuredAdapter` and the `spatialRDD.spatialPartitioningWithoutDu
=== "Python"
```python
- from sedona.utils.structured_adapter import StructuredAdapter
+ from sedona.spark import StructuredAdapter
spatialRDD.spatialPartitioningWithoutDuplicates(GridType.KDBTREE)
# Specify the desired number of partitions as 10, though the actual number may vary
@@ -1427,7 +1427,7 @@ PairRDD is the result of a spatial join query or distance join query. SedonaSQL
=== "Python"
```python
- from sedona.utils.adapter import StructuredAdapter
+ from sedona.spark import StructuredAdapter
joinResultDf = StructuredAdapter.pairRddToDf(result_pair_rdd, leftDf.schema, rightDf.schema, spark)
```
diff --git a/docs/usecases/ApacheSedonaCore.ipynb b/docs/usecases/ApacheSedonaCore.ipynb
index 3f04021844..50270af244 100644
--- a/docs/usecases/ApacheSedonaCore.ipynb
+++ b/docs/usecases/ApacheSedonaCore.ipynb
@@ -40,7 +40,7 @@
"from shapely.geometry import Polygon\n",
"\n",
"from sedona.spark import *\n",
- "from sedona.core.geom.envelope import Envelope"
+ "from sedona.spark import Envelope"
]
},
{
@@ -601,9 +601,7 @@
"|testattribute0|testattribute1|testattribute2|POINT (-88.388954 32.357073)|\n",
"|testattribute0|testattribute1|testattribute2|POINT (-88.221102 32.35078) |\n",
"|testattribute0|testattribute1|testattribute2|POINT (-88.323995 32.950671)|\n",
- "+--------------+--------------+--------------+----------------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------+--------------+--------------+----------------------------+\n"
]
},
{
@@ -1207,9 +1205,7 @@
"|POLYGON ((-87.229...|POINT (-87.19351 ...|\n",
"|POLYGON ((-87.285...|POINT (-87.28468 ...|\n",
"|POLYGON ((-87.285...|POINT (-87.215491...|\n",
- "+--------------------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------------------+\n"
]
},
{
@@ -1246,8 +1242,7 @@
"text": [
"root\n",
" |-- geom_left: geometry (nullable = false)\n",
- " |-- geom_right: geometry (nullable = false)\n",
- "\n"
+ " |-- geom_right: geometry (nullable = false)\n"
]
}
],
@@ -1286,9 +1281,7 @@
"|POLYGON ((-87.229...| |POINT (-87.19351 ...| |\n",
"|POLYGON ((-87.285...| |POINT (-87.28468 ...| |\n",
"|POLYGON ((-87.285...| |POINT (-87.215491...| |\n",
- "+--------------------+-----+--------------------+-----+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+-----+--------------------+-----+\n"
]
},
{
@@ -1330,8 +1323,7 @@
" |-- geom_1: geometry (nullable = true)\n",
" |-- attr1: string (nullable = true)\n",
" |-- geom_2: geometry (nullable = true)\n",
- " |-- attr2: string (nullable = true)\n",
- "\n"
+ " |-- attr2: string (nullable = true)\n"
]
},
{
@@ -1437,8 +1429,7 @@
"|POLYGON ((-87.105...| 15|\n",
"|POLYGON ((-86.860...| 12|\n",
"|POLYGON ((-87.092...| 5|\n",
- "+--------------------+----------------+\n",
- "\n"
+ "+--------------------+----------------+\n"
]
},
{
@@ -1616,7 +1607,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from sedona.core.geom.envelope import Envelope"
+ "from sedona.spark.core.geom.envelope import Envelope"
]
},
{
@@ -1712,9 +1703,7 @@
"|LINESTRING (-72.4...|\n",
"|LINESTRING (-73.4...|\n",
"|LINESTRING (-73.6...|\n",
- "+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+\n"
]
},
{
@@ -1804,9 +1793,7 @@
"|MULTIPOLYGON (((1...|\n",
"|POLYGON ((118.362...|\n",
"|MULTIPOLYGON (((-...|\n",
- "+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+\n"
]
}
],
@@ -1885,9 +1872,7 @@
"|POLYGON ((-86.000...| 01| 055| 001300| 3|1500000US01055001...|010550013003| 3| BG| 1378742|\n",
"|POLYGON ((-86.574...| 01| 089| 001700| 2|1500000US01089001...|010890017002| 2| BG| 1040641|\n",
"|POLYGON ((-85.382...| 01| 069| 041400| 1|1500000US01069041...|010690414001| 1| BG| 8243574|\n",
- "+--------------------+-------+--------+-------+--------+--------------------+------------+----+----+--------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+-------+--------+-------+--------+--------------------+------------+----+----+--------+\n"
]
},
{
@@ -1950,8 +1935,7 @@
"output_type": "stream",
"text": [
"root\n",
- " |-- geometry: geometry (nullable = true)\n",
- "\n"
+ " |-- geometry: geometry (nullable = true)\n"
]
}
],
@@ -1983,9 +1967,7 @@
"|POLYGON ((-104.56...|\n",
"|POLYGON ((-96.910...|\n",
"|POLYGON ((-98.273...|\n",
- "+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+\n"
]
},
{
@@ -2035,9 +2017,7 @@
"|POLYGON ((-104.56...|\n",
"|POLYGON ((-96.910...|\n",
"|POLYGON ((-98.273...|\n",
- "+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+\n"
]
},
{
@@ -2094,8 +2074,7 @@
"text": [
"root\n",
" |-- leftgeometry: geometry (nullable = true)\n",
- " |-- rightgeometry: geometry (nullable = true)\n",
- "\n"
+ " |-- rightgeometry: geometry (nullable = true)\n"
]
}
],
@@ -2127,9 +2106,7 @@
"|POLYGON ((-86.749...|POINT (-86.735506...|\n",
"|POLYGON ((-86.749...|POINT (-86.68645 ...|\n",
"|POLYGON ((-86.749...|POINT (-86.675405...|\n",
- "+--------------------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------------------+\n"
]
}
],
@@ -2190,9 +2167,7 @@
"|POLYGON ((-86.749...| |POINT (-86.735506...| null|\n",
"|POLYGON ((-86.749...| |POINT (-86.68645 ...| null|\n",
"|POLYGON ((-86.749...| |POINT (-86.675405...| null|\n",
- "+--------------------+--------------+--------------------+---------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------------+--------------------+---------------+\n"
]
}
],
@@ -2247,9 +2222,7 @@
"|LINESTRING (-72.4...|\n",
"|LINESTRING (-73.4...|\n",
"|LINESTRING (-73.6...|\n",
- "+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+\n"
]
}
],
@@ -2267,8 +2240,7 @@
"output_type": "stream",
"text": [
"root\n",
- " |-- geometry: geometry (nullable = true)\n",
- "\n"
+ " |-- geometry: geometry (nullable = true)\n"
]
}
],
@@ -2304,9 +2276,7 @@
"|LINESTRING (-72.4...| |\n",
"|LINESTRING (-73.4...| |\n",
"|LINESTRING (-73.6...| |\n",
- "+--------------------+---+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+---+\n"
]
}
],
@@ -2325,8 +2295,7 @@
"text": [
"root\n",
" |-- geometry: geometry (nullable = true)\n",
- " |-- _c1: string (nullable = true)\n",
- "\n"
+ " |-- _c1: string (nullable = true)\n"
]
}
],
diff --git a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
index 1472c2bab3..464bd481de 100644
--- a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
+++ b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
@@ -138,7 +138,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from sedona.utils.structured_adapter import StructuredAdapter\n",
+ "from sedona.spark import StructuredAdapter\n",
"\n",
"airports_rdd = StructuredAdapter.toSpatialRdd(airports_df, \"geometry\")\n",
"# Drop the duplicate name column in countries_df\n",
diff --git a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
index 13d4ea7f72..2c03dd11f2 100644
--- a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
+++ b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
@@ -62,11 +62,11 @@
" ArrayType,\n",
")\n",
"from pyspark.sql.functions import regexp_replace\n",
- "from sedona.register import SedonaRegistrator\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaRegistrator\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit"
]
@@ -181,8 +181,7 @@
" | |-- width: integer (nullable = true)\n",
" | |-- nBands: integer (nullable = true)\n",
" | |-- data: array (nullable = true)\n",
- " | | |-- element: double (containsNull = true)\n",
- "\n"
+ " | | |-- element: double (containsNull = true)\n"
]
}
],
diff --git a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
index c1614ae75d..d5594059ee 100644
--- a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
+++ b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
@@ -56,11 +56,11 @@
" ArrayType,\n",
")\n",
"from pyspark.sql.functions import regexp_replace\n",
- "from sedona.register import SedonaRegistrator\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaRegistrator\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit, flatten\n",
"from pywebhdfs.webhdfs import PyWebHdfsClient\n",
@@ -237,8 +237,7 @@
" | |-- width: integer (nullable = true)\n",
" | |-- nBands: integer (nullable = true)\n",
" | |-- data: array (nullable = true)\n",
- " | | |-- element: double (containsNull = true)\n",
- "\n"
+ " | | |-- element: double (containsNull = true)\n"
]
},
{
@@ -269,7 +268,6 @@
"+- InMemoryTableScan [image#14]\n",
" +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n",
" +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<image:struct<origin:string,wkt:string,height:int,width:int,nBands:int,data:array<double>>>\n",
- "\n",
"\n"
]
},
@@ -292,9 +290,7 @@
"|[hdfs://776faf4d6...| 2|\n",
"|[hdfs://776faf4d6...| 3|\n",
"|[hdfs://776faf4d6...| 4|\n",
- "+--------------------+---+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+---+\n"
]
},
{
@@ -331,7 +327,6 @@
" +- InMemoryTableScan [image#14]\n",
" +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n",
" +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<image:struct<origin:string,wkt:string,height:int,width:int,nBands:int,data:array<double>>>\n",
- "\n",
"\n"
]
}
@@ -428,7 +423,6 @@
" +- InMemoryTableScan [image#14]\n",
" +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n",
" +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<image:struct<origin:string,wkt:string,height:int,width:int,nBands:int,data:array<double>>>\n",
- "\n",
"\n"
]
}
@@ -473,9 +467,7 @@
"|hdfs://776faf4d6a...|POLYGON ((-54.274...|[931.0, 971.0, 95...|[1282.0, 1356.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
"|hdfs://776faf4d6a...|POLYGON ((-54.274...|[957.0, 995.0, 97...|[1282.0, 1354.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
"|hdfs://776faf4d6a...|POLYGON ((-54.274...|[428.0, 428.0, 43...|[880.0, 874.0, 79...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
- "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n"
]
},
{
@@ -530,9 +522,7 @@
"|2021-12-26 13:42:12| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[931.0, 971.0, 95...|[1282.0, 1356.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
"|2021-12-26 13:42:12| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[957.0, 995.0, 97...|[1282.0, 1354.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
"|2021-12-16 13:42:11| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[428.0, 428.0, 43...|[880.0, 874.0, 79...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n",
- "+-------------------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+-------------------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n"
]
},
{
@@ -662,8 +652,7 @@
" | |-- element: double (containsNull = true)\n",
" |-- origin: string (nullable = true)\n",
" |-- image_date: timestamp (nullable = true)\n",
- " |-- feature_name: string (nullable = true)\n",
- "\n"
+ " |-- feature_name: string (nullable = true)\n"
]
},
{
@@ -747,8 +736,7 @@
" |-- tgi: double (nullable = false)\n",
" |-- origin: string (nullable = true)\n",
" |-- image_date: timestamp (nullable = true)\n",
- " |-- feature_name: string (nullable = true)\n",
- "\n"
+ " |-- feature_name: string (nullable = true)\n"
]
}
],
@@ -883,8 +871,7 @@
"| 0.0| 0.0|0.0|-50.85|1.0| 3|(5,[3,4],[-50.85,...|\n",
"| 0.0| 0.0|0.0| -23.8|1.0| 3|(5,[3,4],[-23.8,1...|\n",
"+----+-----+---+------+---+------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "only showing top 5 rows\n"
]
}
],
diff --git a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
index 9867987395..4042c9f5c1 100644
--- a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
+++ b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
@@ -44,11 +44,11 @@
" ArrayType,\n",
")\n",
"from pyspark.sql.functions import regexp_replace\n",
- "from sedona.register import SedonaRegistrator\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaRegistrator\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"from pyspark.sql.functions import col, split, expr\n",
"from pyspark.sql.functions import udf, lit, flatten\n",
"from pywebhdfs.webhdfs import PyWebHdfsClient\n",
@@ -314,8 +314,7 @@
" | |-- copyright: string (nullable = true)\n",
" | |-- timestamp_areas_base: timestamp (nullable = true)\n",
" | |-- timestamp_osm_base: timestamp (nullable = true)\n",
- " |-- version: double (nullable = true)\n",
- "\n"
+ " |-- version: double (nullable = true)\n"
]
}
],
@@ -369,9 +368,7 @@
"|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122631|\n",
"|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122645|\n",
"|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122801|\n",
- "+--------------------+--------------------+--------------------+-------+-----------+--------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------------------+--------------------+-------+-----------+--------+\n"
]
},
{
@@ -442,9 +439,7 @@
"|LINESTRING (-25.5...|26122631| 60| null|concrete|Ponte Internacion...|[285975617, 69310...| 36560|\n",
"|LINESTRING (-25.6...|26122645| null| null| paved|Rodovia das Catar...|[307420349, 16982...| 36560|\n",
"|LINESTRING (-25.5...|26122801| null| null| null| Rua Natal Graciotin|[5514364259, 5514...| 36560|\n",
- "+--------------------+--------+--------+-------+--------+--------------------+--------------------+-----------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+--------+--------+-------+--------+--------------------+--------------------+-----------+\n"
]
}
],
@@ -616,9 +611,7 @@
"|[285975617, 69310...| 118.8| 0.00| 0.02| 60| 0%|concrete|Ponte Internacion...|26122631|LINESTRING (-25.5...|1.9538901610218935|0.12057253982300521|0.10941553855472455| 36560|\n",
"|[307420349, 16982...| 39.4| 0.00| 0.03| 20| 0%| paved|Rodovia das Catar...|26122645|LINESTRING (-25.6...|11.873439986567648| 0.2619862588434014|0.24791409977758808| 36560|\n",
"|[5514364259, 5514...| 39.2| 0.00| 0.04| 20| 0%| soil| Rua Natal Graciotin|26122801|LINESTRING (-25.5...| 6.558866347858073|0.08873043404159503| 0.079155366559698| 36560|\n",
- "+--------------------+------+-------------+-------------+--------+-------+--------+--------------------+--------+--------------------+------------------+-------------------+-------------------+-----------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+--------------------+------+-------------+-------------+--------+-------+--------+--------------------+--------+--------------------+------------------+-------------------+-------------------+-----------+\n"
]
}
],
@@ -687,8 +680,7 @@
"|min(weight)|\n",
"+-----------+\n",
"| 39.0|\n",
- "+-----------+\n",
- "\n"
+ "+-----------+\n"
]
}
],
@@ -752,8 +744,7 @@
"| id| distance|\n",
"+--------+--------------------+\n",
"|32347167|8.330276767022637E-5|\n",
- "+--------+--------------------+\n",
- "\n"
+ "+--------+--------------------+\n"
]
},
{
@@ -848,9 +839,7 @@
"|918520467|LINESTRING (-25.4...|\n",
"| 32345349|LINESTRING (-25.4...|\n",
"|964560305|LINESTRING (-25.4...|\n",
- "+---------+--------------------+\n",
- "only showing top 5 rows\n",
- "\n"
+ "+---------+--------------------+\n"
]
}
],
diff --git a/docs/usecases/contrib/foot-traffic.ipynb b/docs/usecases/contrib/foot-traffic.ipynb
index 200da0dc72..f1ee331c3a 100644
--- a/docs/usecases/contrib/foot-traffic.ipynb
+++ b/docs/usecases/contrib/foot-traffic.ipynb
@@ -111,8 +111,8 @@
],
"source": [
"from pyspark.sql import SparkSession\n",
- "from sedona.register import SedonaRegistrator\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaRegistrator\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"\n",
"spark = (\n",
" SparkSession.builder.appName(\"sigspatial2021\")\n",
@@ -794,9 +794,7 @@
"|zzw-22d@5x4-4b7-bkz|Eltana |305 Harrison St |294.919 |\n",
"|zzw-224@5x4-4b5-f75|MarketSpice |85A Pike St |263.49886363636364 |\n",
"|22b-222@5x4-4vs-qzz|Cherry Street Coffee House|700 1st Ave |210.19386666666668 |\n",
- "+-------------------+--------------------------+-----------------------------------+-------------------------+\n",
- "only showing top 20 rows\n",
- "\n"
+ "+-------------------+--------------------------+-----------------------------------+-------------------------+\n"
]
},
{
@@ -898,8 +896,8 @@
"# but `distance_from_home` takes into account ALL visitors to the coffee shop, which as we saw above can distort values. When selecting a site for a coffee shop, we likely care more about how far visitors traveled from within Seattle\n",
"# we can compute this using Sedona\n",
"import sedona\n",
- "from sedona.register import SedonaRegistrator\n",
- "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
+ "from sedona.spark import SedonaRegistrator\n",
+ "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n",
"\n",
"SedonaRegistrator.registerAll(spark)"
]
diff --git a/python/pytest.ini b/python/pytest.ini
new file mode 100644
index 0000000000..0e30126897
--- /dev/null
+++ b/python/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+markers =
+ flink: Apache Flink based tests
diff --git a/python/sedona/flink/__init__.py b/python/sedona/flink/__init__.py
new file mode 100644
index 0000000000..4219449814
--- /dev/null
+++ b/python/sedona/flink/__init__.py
@@ -0,0 +1,14 @@
+import logging
+import warnings
+
+try:
+ import pyflink
+ from sedona.flink.context import SedonaContext
+except ImportError:
+ warnings.warn(
+ "Apache Sedona requires Pyflink. Please install PyFlink before using Sedona flink.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+__all__ = ["SedonaContext"]
diff --git a/python/sedona/flink/context.py b/python/sedona/flink/context.py
new file mode 100644
index 0000000000..2f14d20b19
--- /dev/null
+++ b/python/sedona/flink/context.py
@@ -0,0 +1,35 @@
+import warnings
+
+try:
+ from pyflink.table import EnvironmentSettings, StreamTableEnvironment
+ from pyflink.datastream import StreamExecutionEnvironment
+ from pyflink.java_gateway import get_gateway
+except ImportError:
+ StreamTableEnvironment = None
+ StreamExecutionEnvironment = None
+ EnvironmentSettings = None
+ warnings.warn(
+ "Apache Sedona requires Pyflink. Please install PyFlink before using Sedona flink.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+
+class SedonaContext:
+
+ @classmethod
+ def create(
+ cls, env: StreamExecutionEnvironment, settings: EnvironmentSettings
+ ) -> StreamTableEnvironment:
+ table_env = StreamTableEnvironment.create(env, settings)
+ gateway = get_gateway()
+
+ flink_sedona_context = gateway.jvm.org.apache.sedona.flink.SedonaContext
+
+ table_env_j = flink_sedona_context.create(
+ env._j_stream_execution_environment, table_env._j_tenv
+ )
+
+ table_env._j_tenv = table_env_j
+
+ return table_env
diff --git a/python/sedona/raster_utils/__init__.py b/python/sedona/raster_utils/__init__.py
deleted file mode 100644
index a67d5ea255..0000000000
--- a/python/sedona/raster_utils/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/python/sedona/spark/SedonaContext.py b/python/sedona/spark/SedonaContext.py
index 49db2e47aa..5a8dc97330 100644
--- a/python/sedona/spark/SedonaContext.py
+++ b/python/sedona/spark/SedonaContext.py
@@ -18,8 +18,8 @@
import attr
from pyspark.sql import SparkSession
-from sedona.register.geo_registrator import PackageImporter
-from sedona.utils import KryoSerializer, SedonaKryoRegistrator
+from sedona.spark.register.geo_registrator import PackageImporter
+from sedona.spark.utils import KryoSerializer, SedonaKryoRegistrator
try:
from pyspark.sql.utils import is_remote
diff --git a/python/sedona/spark/__init__.py b/python/sedona/spark/__init__.py
index 8f4c4f24ed..15358904db 100644
--- a/python/sedona/spark/__init__.py
+++ b/python/sedona/spark/__init__.py
@@ -14,18 +14,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from sedona.spark.core import Envelope
+from sedona.spark.sql.functions import sedona_vectorized_udf
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.formatMapper import GeoJsonReader, WkbReader, WktReader
-from sedona.core.formatMapper.shapefileParser import ShapefileReader
-from sedona.core.spatialOperator import (
+try:
+ import pyspark
+except ImportError:
+ raise ImportError(
+ "Apache Sedona requires PySpark. Please install PySpark before using Sedona spark."
+ )
+
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.formatMapper import GeoJsonReader, WkbReader, WktReader
+from sedona.spark.core.formatMapper.shapefileParser import ShapefileReader
+from sedona.spark.core.spatialOperator import (
JoinQuery,
JoinQueryRaw,
KNNQuery,
RangeQuery,
RangeQueryRaw,
)
-from sedona.core.SpatialRDD import (
+from sedona.spark.core.SpatialRDD import (
CircleRDD,
LineStringRDD,
PointRDD,
@@ -33,16 +42,17 @@ from sedona.core.SpatialRDD import (
RectangleRDD,
SpatialRDD,
)
-from sedona.maps.SedonaKepler import SedonaKepler
-from sedona.maps.SedonaPyDeck import SedonaPyDeck
-from sedona.raster_utils.SedonaUtils import SedonaUtils
-from sedona.register import SedonaRegistrator
+from sedona.spark.maps.SedonaKepler import SedonaKepler
+from sedona.spark.maps.SedonaPyDeck import SedonaPyDeck
+from sedona.spark.raster_utils.SedonaUtils import SedonaUtils
+from sedona.spark.register import SedonaRegistrator
from sedona.spark.SedonaContext import SedonaContext
-from sedona.sql.st_aggregates import *
-from sedona.sql.st_constructors import *
-from sedona.sql.st_functions import *
-from sedona.sql.st_predicates import *
-from sedona.sql.types import GeometryType, GeographyType, RasterType
-from sedona.utils import KryoSerializer, SedonaKryoRegistrator
-from sedona.utils.adapter import Adapter
-from sedona.utils.geoarrow import dataframe_to_arrow
+from sedona.spark.sql.st_aggregates import *
+from sedona.spark.sql.st_constructors import *
+from sedona.spark.sql.st_functions import *
+from sedona.spark.sql.st_predicates import *
+from sedona.spark.sql.types import GeometryType, GeographyType, RasterType
+from sedona.spark.utils import KryoSerializer, SedonaKryoRegistrator
+from sedona.spark.utils.adapter import Adapter
+from sedona.spark.utils.structured_adapter import StructuredAdapter
+from sedona.spark.geoarrow.geoarrow import dataframe_to_arrow
diff --git a/python/sedona/core/SpatialRDD/__init__.py b/python/sedona/spark/core/SpatialRDD/__init__.py
similarity index 100%
rename from python/sedona/core/SpatialRDD/__init__.py
rename to python/sedona/spark/core/SpatialRDD/__init__.py
diff --git a/python/sedona/core/SpatialRDD/circle_rdd.py b/python/sedona/spark/core/SpatialRDD/circle_rdd.py
similarity index 84%
rename from python/sedona/core/SpatialRDD/circle_rdd.py
rename to python/sedona/spark/core/SpatialRDD/circle_rdd.py
index 48142a52a8..6cd13a1000 100644
--- a/python/sedona/core/SpatialRDD/circle_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/circle_rdd.py
@@ -15,9 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.meta import MultipleMeta
class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
@@ -32,7 +32,7 @@ class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
self._srdd = self._jvm_spatial_rdd(spatialRDD._srdd, Radius)
def getCenterPointAsSpatialRDD(self) -> "PointRDD":
- from sedona.core.SpatialRDD import PointRDD
+ from sedona.spark.core.SpatialRDD import PointRDD
srdd = self._srdd.getCenterPointAsSpatialRDD()
point_rdd = PointRDD()
@@ -40,7 +40,7 @@ class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
return point_rdd
def getCenterPolygonAsSpatialRDD(self) -> "PolygonRDD":
- from sedona.core.SpatialRDD import PolygonRDD
+ from sedona.spark.core.SpatialRDD import PolygonRDD
srdd = self._srdd.getCenterPolygonAsSpatialRDD()
polygon_rdd = PolygonRDD()
@@ -48,7 +48,7 @@ class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
return polygon_rdd
def getCenterLineStringRDDAsSpatialRDD(self) -> "LineStringRDD":
- from sedona.core.SpatialRDD import LineStringRDD
+ from sedona.spark.core.SpatialRDD import LineStringRDD
srdd = self._srdd.getCenterPolygonAsSpatialRDD()
linestring_rdd = LineStringRDD()
@@ -56,7 +56,7 @@ class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
return linestring_rdd
def getCenterRectangleRDDAsSpatialRDD(self) -> "RectangleRDD":
- from sedona.core.SpatialRDD import RectangleRDD
+ from sedona.spark.core.SpatialRDD import RectangleRDD
srdd = self._srdd.getCenterLineStringRDDAsSpatialRDD()
rectangle_rdd = RectangleRDD()
diff --git a/python/sedona/core/SpatialRDD/linestring_rdd.py b/python/sedona/spark/core/SpatialRDD/linestring_rdd.py
similarity index 91%
rename from python/sedona/core/SpatialRDD/linestring_rdd.py
rename to python/sedona/spark/core/SpatialRDD/linestring_rdd.py
index fcdf9450ec..7d5e55fdda 100644
--- a/python/sedona/core/SpatialRDD/linestring_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/linestring_rdd.py
@@ -15,14 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-from pyspark import RDD, SparkContext, StorageLevel
+from pyspark import RDD, SparkContext
-from sedona.core.enums import FileDataSplitter
-from sedona.core.enums.file_data_splitter import FileSplitterJvm
-from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter
-from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.enums.file_data_splitter import FileSplitterJvm
+from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter
+from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.meta import MultipleMeta
class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
@@ -166,7 +166,7 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
return spatial_factory.create_linestring_rdd()
def MinimumBoundingRectangle(self):
- from sedona.core.SpatialRDD import RectangleRDD
+ from sedona.spark.core.SpatialRDD import RectangleRDD
rectangle_rdd = RectangleRDD()
srdd = self._srdd.MinimumBoundingRectangle()
diff --git a/python/sedona/core/SpatialRDD/point_rdd.py b/python/sedona/spark/core/SpatialRDD/point_rdd.py
similarity index 92%
rename from python/sedona/core/SpatialRDD/point_rdd.py
rename to python/sedona/spark/core/SpatialRDD/point_rdd.py
index 54205cad60..fade34efec 100644
--- a/python/sedona/core/SpatialRDD/point_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/point_rdd.py
@@ -15,13 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-from pyspark import RDD, SparkContext, StorageLevel
+from pyspark import RDD, SparkContext
-from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
-from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter
-from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
+from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter
+from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.meta import MultipleMeta
class PointRDD(SpatialRDD, metaclass=MultipleMeta):
diff --git a/python/sedona/core/SpatialRDD/polygon_rdd.py b/python/sedona/spark/core/SpatialRDD/polygon_rdd.py
similarity index 91%
rename from python/sedona/core/SpatialRDD/polygon_rdd.py
rename to python/sedona/spark/core/SpatialRDD/polygon_rdd.py
index ab6dfa4637..e4f7ae4020 100644
--- a/python/sedona/core/SpatialRDD/polygon_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/polygon_rdd.py
@@ -15,13 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-from pyspark import RDD, SparkContext, StorageLevel
+from pyspark import RDD, SparkContext
-from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
-from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter
-from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
+from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter
+from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.meta import MultipleMeta
class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
@@ -161,7 +161,7 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
)
def MinimumBoundingRectangle(self):
- from sedona.core.SpatialRDD import RectangleRDD
+ from sedona.spark.core.SpatialRDD import RectangleRDD
rectangle_rdd = RectangleRDD()
srdd = self._srdd.MinimumBoundingRectangle()
diff --git a/python/sedona/core/SpatialRDD/rectangle_rdd.py b/python/sedona/spark/core/SpatialRDD/rectangle_rdd.py
similarity index 92%
rename from python/sedona/core/SpatialRDD/rectangle_rdd.py
rename to python/sedona/spark/core/SpatialRDD/rectangle_rdd.py
index 3d8e457ef2..397cdee453 100644
--- a/python/sedona/core/SpatialRDD/rectangle_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/rectangle_rdd.py
@@ -15,13 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-from pyspark import RDD, SparkContext, StorageLevel
+from pyspark import SparkContext
-from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
-from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.jvm import JvmStorageLevel
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm
+from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.meta import MultipleMeta
class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
diff --git a/python/sedona/core/SpatialRDD/spatial_rdd.py b/python/sedona/spark/core/SpatialRDD/spatial_rdd.py
similarity index 95%
rename from python/sedona/core/SpatialRDD/spatial_rdd.py
rename to python/sedona/spark/core/SpatialRDD/spatial_rdd.py
index 266676c0d5..f091c49003 100644
--- a/python/sedona/core/SpatialRDD/spatial_rdd.py
+++ b/python/sedona/spark/core/SpatialRDD/spatial_rdd.py
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-import pickle
from typing import List, Optional, Union
import attr
@@ -23,16 +22,19 @@ from py4j.java_gateway import get_field, get_method
from pyspark import RDD, SparkContext, StorageLevel
from pyspark.sql import SparkSession
-from sedona.core.enums.grid_type import GridType, GridTypeJvm
-from sedona.core.enums.index_type import IndexType, IndexTypeJvm
-from sedona.core.enums.spatial import SpatialType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.jvm.translate import JvmSedonaPythonConverter, SedonaPythonConverter
-from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
-from sedona.utils.decorators import require
-from sedona.utils.jvm import JvmStorageLevel
-from sedona.utils.spatial_rdd_parser import SedonaPickler
-from sedona.utils.types import crs
+from sedona.spark.core.enums.grid_type import GridType, GridTypeJvm
+from sedona.spark.core.enums.index_type import IndexType, IndexTypeJvm
+from sedona.spark.core.enums.spatial import SpatialType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.jvm.translate import (
+ JvmSedonaPythonConverter,
+ SedonaPythonConverter,
+)
+from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory
+from sedona.spark.utils.decorators import require
+from sedona.spark.utils.jvm import JvmStorageLevel
+from sedona.spark.utils.spatial_rdd_parser import SedonaPickler
+from sedona.spark.utils.types import crs
@attr.s
@@ -404,6 +406,8 @@ class SpatialRDD:
@property
def spatialPartitionedRDD(self):
+ from sedona.spark.utils.spatial_rdd_parser import SedonaPickler
+
"""
:return:
diff --git a/python/sedona/core/SpatialRDD/spatial_rdd_factory.py b/python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py
similarity index 97%
rename from python/sedona/core/SpatialRDD/spatial_rdd_factory.py
rename to python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py
index f44611ea37..ad504ab9e5 100644
--- a/python/sedona/core/SpatialRDD/spatial_rdd_factory.py
+++ b/python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py
@@ -20,7 +20,7 @@ from abc import ABC
import attr
from pyspark import SparkContext
-from sedona.utils.decorators import require
+from sedona.spark.utils.decorators import require
@attr.s
diff --git a/python/sedona/utils/__init__.py b/python/sedona/spark/core/__init__.py
similarity index 83%
copy from python/sedona/utils/__init__.py
copy to python/sedona/spark/core/__init__.py
index 8c3ca09339..54cae81f09 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/spark/core/__init__.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from .serde import KryoSerializer, SedonaKryoRegistrator
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQueryRaw
-__all__ = ["KryoSerializer", "SedonaKryoRegistrator"]
+__all__ = ["Envelope", "RangeQueryRaw"]
diff --git a/python/sedona/core/enums/__init__.py b/python/sedona/spark/core/enums/__init__.py
similarity index 100%
rename from python/sedona/core/enums/__init__.py
rename to python/sedona/spark/core/enums/__init__.py
diff --git a/python/sedona/core/enums/file_data_splitter.py b/python/sedona/spark/core/enums/file_data_splitter.py
similarity index 94%
rename from python/sedona/core/enums/file_data_splitter.py
rename to python/sedona/spark/core/enums/file_data_splitter.py
index 81b402b3f3..a247092290 100644
--- a/python/sedona/core/enums/file_data_splitter.py
+++ b/python/sedona/spark/core/enums/file_data_splitter.py
@@ -19,8 +19,8 @@ from enum import Enum
import attr
-from sedona.core.jvm.abstract import JvmObject
-from sedona.utils.decorators import require
+from sedona.spark.core.jvm.abstract import JvmObject
+from sedona.spark.utils.decorators import require
class FileDataSplitter(Enum):
diff --git a/python/sedona/core/enums/grid_type.py b/python/sedona/spark/core/enums/grid_type.py
similarity index 93%
rename from python/sedona/core/enums/grid_type.py
rename to python/sedona/spark/core/enums/grid_type.py
index 89deeee964..8d43c44737 100644
--- a/python/sedona/core/enums/grid_type.py
+++ b/python/sedona/spark/core/enums/grid_type.py
@@ -19,8 +19,8 @@ from enum import Enum
import attr
-from sedona.core.jvm.abstract import JvmObject
-from sedona.utils.decorators import require
+from sedona.spark.core.jvm.abstract import JvmObject
+from sedona.spark.utils.decorators import require
class GridType(Enum):
diff --git a/python/sedona/core/enums/index_type.py b/python/sedona/spark/core/enums/index_type.py
similarity index 93%
rename from python/sedona/core/enums/index_type.py
rename to python/sedona/spark/core/enums/index_type.py
index 3a00004d84..dc29d696a2 100644
--- a/python/sedona/core/enums/index_type.py
+++ b/python/sedona/spark/core/enums/index_type.py
@@ -19,8 +19,8 @@ from enum import Enum
import attr
-from sedona.core.jvm.abstract import JvmObject
-from sedona.utils.decorators import require
+from sedona.spark.core.jvm.abstract import JvmObject
+from sedona.spark.utils.decorators import require
class IndexType(Enum):
diff --git a/python/sedona/core/enums/join_build_side.py b/python/sedona/spark/core/enums/join_build_side.py
similarity index 94%
rename from python/sedona/core/enums/join_build_side.py
rename to python/sedona/spark/core/enums/join_build_side.py
index f79c159611..d0c5ed6312 100644
--- a/python/sedona/core/enums/join_build_side.py
+++ b/python/sedona/spark/core/enums/join_build_side.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from sedona.utils.decorators import classproperty
+from sedona.spark.utils.decorators import classproperty
class JoinBuildSide:
diff --git a/python/sedona/core/enums/spatial.py b/python/sedona/spark/core/enums/spatial.py
similarity index 100%
rename from python/sedona/core/enums/spatial.py
rename to python/sedona/spark/core/enums/spatial.py
diff --git a/python/sedona/core/formatMapper/__init__.py b/python/sedona/spark/core/formatMapper/__init__.py
similarity index 100%
rename from python/sedona/core/formatMapper/__init__.py
rename to python/sedona/spark/core/formatMapper/__init__.py
diff --git a/python/sedona/core/formatMapper/disc_utils.py b/python/sedona/spark/core/formatMapper/disc_utils.py
similarity index 94%
rename from python/sedona/core/formatMapper/disc_utils.py
rename to python/sedona/spark/core/formatMapper/disc_utils.py
index e5b8256d35..459e864ad4 100644
--- a/python/sedona/core/formatMapper/disc_utils.py
+++ b/python/sedona/spark/core/formatMapper/disc_utils.py
@@ -19,9 +19,9 @@ from enum import Enum
from pyspark import SparkContext
-from sedona.core.jvm.translate import SpatialObjectLoaderAdapter
-from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, SpatialRDD
-from sedona.utils.decorators import require
+from sedona.spark.core.jvm.translate import SpatialObjectLoaderAdapter
+from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, SpatialRDD
+from sedona.spark.utils.decorators import require
class DiscLoader:
diff --git a/python/sedona/core/formatMapper/geo_json_reader.py b/python/sedona/spark/core/formatMapper/geo_json_reader.py
similarity index 94%
rename from python/sedona/core/formatMapper/geo_json_reader.py
rename to python/sedona/spark/core/formatMapper/geo_json_reader.py
index 7b1784f0cf..23a090cd0c 100644
--- a/python/sedona/core/formatMapper/geo_json_reader.py
+++ b/python/sedona/spark/core/formatMapper/geo_json_reader.py
@@ -17,9 +17,9 @@
from pyspark import RDD, SparkContext
-from sedona.core.formatMapper.geo_reader import GeoDataReader
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.formatMapper.geo_reader import GeoDataReader
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.meta import MultipleMeta
class GeoJsonReader(GeoDataReader, metaclass=MultipleMeta):
diff --git a/python/sedona/core/formatMapper/geo_reader.py b/python/sedona/spark/core/formatMapper/geo_reader.py
similarity index 95%
rename from python/sedona/core/formatMapper/geo_reader.py
rename to python/sedona/spark/core/formatMapper/geo_reader.py
index 1cfb373a97..96fdee509f 100644
--- a/python/sedona/core/formatMapper/geo_reader.py
+++ b/python/sedona/spark/core/formatMapper/geo_reader.py
@@ -19,7 +19,7 @@ import abc
import attr
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.utils.meta import MultipleMeta
@attr.s
diff --git a/python/sedona/core/formatMapper/shapefileParser/__init__.py b/python/sedona/spark/core/formatMapper/shapefileParser/__init__.py
similarity index 100%
rename from python/sedona/core/formatMapper/shapefileParser/__init__.py
rename to python/sedona/spark/core/formatMapper/shapefileParser/__init__.py
diff --git a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py b/python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py
similarity index 90%
rename from python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py
rename to python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py
index 4c45609655..936ef5da22 100644
--- a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py
+++ b/python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py
@@ -18,10 +18,10 @@
import attr
from pyspark import SparkContext
-from sedona.core.formatMapper.geo_reader import GeoDataReader
-from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.formatMapper.geo_reader import GeoDataReader
+from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.meta import MultipleMeta
@attr.s
diff --git a/python/sedona/core/formatMapper/wkb_reader.py b/python/sedona/spark/core/formatMapper/wkb_reader.py
similarity index 93%
rename from python/sedona/core/formatMapper/wkb_reader.py
rename to python/sedona/spark/core/formatMapper/wkb_reader.py
index 45bbc5acdb..2fd373b096 100644
--- a/python/sedona/core/formatMapper/wkb_reader.py
+++ b/python/sedona/spark/core/formatMapper/wkb_reader.py
@@ -17,9 +17,9 @@
from pyspark import RDD, SparkContext
-from sedona.core.formatMapper.geo_reader import GeoDataReader
-from sedona.core.SpatialRDD import SpatialRDD
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.formatMapper.geo_reader import GeoDataReader
+from sedona.spark.core.SpatialRDD import SpatialRDD
+from sedona.spark.utils.meta import MultipleMeta
class WkbReader(GeoDataReader, metaclass=MultipleMeta):
diff --git a/python/sedona/core/formatMapper/wkt_reader.py b/python/sedona/spark/core/formatMapper/wkt_reader.py
similarity index 93%
rename from python/sedona/core/formatMapper/wkt_reader.py
rename to python/sedona/spark/core/formatMapper/wkt_reader.py
index 7d5689ecee..6e8ac588d7 100644
--- a/python/sedona/core/formatMapper/wkt_reader.py
+++ b/python/sedona/spark/core/formatMapper/wkt_reader.py
@@ -17,9 +17,9 @@
from pyspark import RDD, SparkContext
-from sedona.core.formatMapper.geo_reader import GeoDataReader
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.formatMapper.geo_reader import GeoDataReader
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.meta import MultipleMeta
class WktReader(GeoDataReader, metaclass=MultipleMeta):
diff --git a/python/sedona/core/__init__.py b/python/sedona/spark/core/geom/__init__.py
similarity index 100%
rename from python/sedona/core/__init__.py
rename to python/sedona/spark/core/geom/__init__.py
diff --git a/python/sedona/core/geom/circle.py b/python/sedona/spark/core/geom/circle.py
similarity index 100%
rename from python/sedona/core/geom/circle.py
rename to python/sedona/spark/core/geom/circle.py
diff --git a/python/sedona/core/geom/envelope.py b/python/sedona/spark/core/geom/envelope.py
similarity index 100%
rename from python/sedona/core/geom/envelope.py
rename to python/sedona/spark/core/geom/envelope.py
diff --git a/python/sedona/core/geom/geography.py b/python/sedona/spark/core/geom/geography.py
similarity index 100%
rename from python/sedona/core/geom/geography.py
rename to python/sedona/spark/core/geom/geography.py
diff --git a/python/sedona/core/geom/__init__.py b/python/sedona/spark/core/geom/shapely1/__init__.py
similarity index 100%
rename from python/sedona/core/geom/__init__.py
rename to python/sedona/spark/core/geom/shapely1/__init__.py
diff --git a/python/sedona/core/geom/shapely1/circle.py b/python/sedona/spark/core/geom/shapely1/circle.py
similarity index 98%
rename from python/sedona/core/geom/shapely1/circle.py
rename to python/sedona/spark/core/geom/shapely1/circle.py
index f227e748c7..bce29c703a 100644
--- a/python/sedona/core/geom/shapely1/circle.py
+++ b/python/sedona/spark/core/geom/shapely1/circle.py
@@ -27,7 +27,7 @@ from shapely.geometry import (
)
from shapely.geometry.base import BaseGeometry
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.geom.envelope import Envelope
class Circle(Polygon):
diff --git a/python/sedona/core/geom/shapely1/envelope.py b/python/sedona/spark/core/geom/shapely1/envelope.py
similarity index 97%
rename from python/sedona/core/geom/shapely1/envelope.py
rename to python/sedona/spark/core/geom/shapely1/envelope.py
index df49e61505..20fa45ff96 100644
--- a/python/sedona/core/geom/shapely1/envelope.py
+++ b/python/sedona/spark/core/geom/shapely1/envelope.py
@@ -18,11 +18,10 @@
import math
import pickle
+from sedona.spark.utils.decorators import require
from shapely.geometry import Point, Polygon
from shapely.geometry.base import BaseGeometry
-from sedona.utils.decorators import require
-
class Envelope(Polygon):
@@ -65,7 +64,7 @@ class Envelope(Polygon):
)
def to_bytes(self):
- from sedona.utils.binary_parser import BinaryBuffer
+ from sedona.spark.utils.binary_parser import BinaryBuffer
bin_buffer = BinaryBuffer()
bin_buffer.put_double(self.minx)
diff --git a/python/sedona/core/geom/shapely1/__init__.py b/python/sedona/spark/core/geom/shapely2/__init__.py
similarity index 100%
copy from python/sedona/core/geom/shapely1/__init__.py
copy to python/sedona/spark/core/geom/shapely2/__init__.py
diff --git a/python/sedona/core/geom/shapely2/circle.py b/python/sedona/spark/core/geom/shapely2/circle.py
similarity index 98%
rename from python/sedona/core/geom/shapely2/circle.py
rename to python/sedona/spark/core/geom/shapely2/circle.py
index 859d1ef3b2..9d91682554 100644
--- a/python/sedona/core/geom/shapely2/circle.py
+++ b/python/sedona/spark/core/geom/shapely2/circle.py
@@ -24,11 +24,10 @@ from shapely.geometry import (
MultiPolygon,
Point,
Polygon,
- box,
)
from shapely.geometry.base import BaseGeometry
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.geom.envelope import Envelope
class Circle(Polygon):
diff --git a/python/sedona/core/geom/shapely2/envelope.py b/python/sedona/spark/core/geom/shapely2/envelope.py
similarity index 97%
rename from python/sedona/core/geom/shapely2/envelope.py
rename to python/sedona/spark/core/geom/shapely2/envelope.py
index 6a5aeb0787..3e17f826ed 100644
--- a/python/sedona/core/geom/shapely2/envelope.py
+++ b/python/sedona/spark/core/geom/shapely2/envelope.py
@@ -21,7 +21,7 @@ import pickle
from shapely.geometry import Polygon, box
from shapely.geometry.base import BaseGeometry
-from sedona.utils.decorators import require
+from sedona.spark.utils.decorators import require
class Envelope(Polygon):
@@ -77,7 +77,7 @@ class Envelope(Polygon):
)
def to_bytes(self):
- from sedona.utils.binary_parser import BinaryBuffer
+ from sedona.spark.utils import BinaryBuffer
minx, miny, maxx, maxy = self.bounds
bin_buffer = BinaryBuffer()
diff --git a/python/sedona/core/geom/shapely2/__init__.py b/python/sedona/spark/core/jvm/__init__.py
similarity index 100%
rename from python/sedona/core/geom/shapely2/__init__.py
rename to python/sedona/spark/core/jvm/__init__.py
diff --git a/python/sedona/core/jvm/abstract.py b/python/sedona/spark/core/jvm/abstract.py
similarity index 100%
rename from python/sedona/core/jvm/abstract.py
rename to python/sedona/spark/core/jvm/abstract.py
diff --git a/python/sedona/core/jvm/config.py b/python/sedona/spark/core/jvm/config.py
similarity index 99%
rename from python/sedona/core/jvm/config.py
rename to python/sedona/spark/core/jvm/config.py
index 09650ed51b..fdbc371e4c 100644
--- a/python/sedona/core/jvm/config.py
+++ b/python/sedona/spark/core/jvm/config.py
@@ -26,7 +26,7 @@ from typing import Optional, Tuple
from py4j.protocol import Py4JJavaError
from pyspark.sql import SparkSession
-from sedona.utils.decorators import classproperty
+from sedona.spark.utils.decorators import classproperty
string_types = (bytes, str)
diff --git a/python/sedona/core/jvm/translate.py b/python/sedona/spark/core/jvm/translate.py
similarity index 100%
rename from python/sedona/core/jvm/translate.py
rename to python/sedona/spark/core/jvm/translate.py
diff --git a/python/sedona/core/spatialOperator/__init__.py b/python/sedona/spark/core/spatialOperator/__init__.py
similarity index 100%
rename from python/sedona/core/spatialOperator/__init__.py
rename to python/sedona/spark/core/spatialOperator/__init__.py
diff --git a/python/sedona/core/spatialOperator/join_params.py b/python/sedona/spark/core/spatialOperator/join_params.py
similarity index 92%
rename from python/sedona/core/spatialOperator/join_params.py
rename to python/sedona/spark/core/spatialOperator/join_params.py
index c77224af7f..f22177af3c 100644
--- a/python/sedona/core/spatialOperator/join_params.py
+++ b/python/sedona/spark/core/spatialOperator/join_params.py
@@ -17,9 +17,9 @@
import attr
-from sedona.core.enums import IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.jvm.abstract import JvmObject
+from sedona.spark.core.enums import IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.jvm.abstract import JvmObject
@attr.s
diff --git a/python/sedona/core/spatialOperator/join_query.py b/python/sedona/spark/core/spatialOperator/join_query.py
similarity index 94%
rename from python/sedona/core/spatialOperator/join_query.py
rename to python/sedona/spark/core/spatialOperator/join_query.py
index 7e1f016536..70830a800e 100644
--- a/python/sedona/core/spatialOperator/join_query.py
+++ b/python/sedona/spark/core/spatialOperator/join_query.py
@@ -17,10 +17,10 @@
from pyspark import RDD
-from sedona.core.spatialOperator.join_params import JoinParams
-from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.decorators import require
+from sedona.spark.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.decorators import require
class JoinQuery:
diff --git a/python/sedona/core/spatialOperator/join_query_raw.py b/python/sedona/spark/core/spatialOperator/join_query_raw.py
similarity index 92%
rename from python/sedona/core/spatialOperator/join_query_raw.py
rename to python/sedona/spark/core/spatialOperator/join_query_raw.py
index e97118215a..fbaace1b2a 100644
--- a/python/sedona/core/spatialOperator/join_query_raw.py
+++ b/python/sedona/spark/core/spatialOperator/join_query_raw.py
@@ -15,10 +15,10 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.core.spatialOperator.join_params import JoinParams
-from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaPairRDDList
-from sedona.core.SpatialRDD import SpatialRDD
-from sedona.utils.decorators import require
+from sedona.spark.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD, SedonaPairRDDList
+from sedona.spark.core.SpatialRDD import SpatialRDD
+from sedona.spark.utils.decorators import require
class JoinQueryRaw:
diff --git a/python/sedona/core/spatialOperator/knn_query.py b/python/sedona/spark/core/spatialOperator/knn_query.py
similarity index 83%
rename from python/sedona/core/spatialOperator/knn_query.py
rename to python/sedona/spark/core/spatialOperator/knn_query.py
index c858cdc0f6..5f68943efb 100644
--- a/python/sedona/core/spatialOperator/knn_query.py
+++ b/python/sedona/spark/core/spatialOperator/knn_query.py
@@ -18,12 +18,12 @@
import attr
from shapely.geometry.base import BaseGeometry
-from sedona.core.jvm.translate import JvmSedonaPythonConverter
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.binary_parser import BinaryParser
-from sedona.utils.decorators import require
-from sedona.utils.geometry_adapter import GeometryAdapter
-from sedona.utils.spatial_rdd_parser import SpatialRDDParserData
+from sedona.spark.core.jvm.translate import JvmSedonaPythonConverter
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.binary_parser import BinaryParser
+from sedona.spark.utils.decorators import require
+from sedona.spark.utils.geometry_adapter import GeometryAdapter
+from sedona.spark.utils.spatial_rdd_parser import SpatialRDDParserData
@attr.s
diff --git a/python/sedona/core/spatialOperator/range_query.py b/python/sedona/spark/core/spatialOperator/range_query.py
similarity index 88%
rename from python/sedona/core/spatialOperator/range_query.py
rename to python/sedona/spark/core/spatialOperator/range_query.py
index 4990f15ab7..2d557cdfe7 100644
--- a/python/sedona/core/spatialOperator/range_query.py
+++ b/python/sedona/spark/core/spatialOperator/range_query.py
@@ -17,9 +17,9 @@
from shapely.geometry.base import BaseGeometry
-from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.decorators import require
+from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.decorators import require
class RangeQuery:
diff --git a/python/sedona/core/spatialOperator/range_query_raw.py b/python/sedona/spark/core/spatialOperator/range_query_raw.py
similarity index 87%
rename from python/sedona/core/spatialOperator/range_query_raw.py
rename to python/sedona/spark/core/spatialOperator/range_query_raw.py
index 1b03ef58bf..c7481da8d5 100644
--- a/python/sedona/core/spatialOperator/range_query_raw.py
+++ b/python/sedona/spark/core/spatialOperator/range_query_raw.py
@@ -17,10 +17,10 @@
from shapely.geometry.base import BaseGeometry
-from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.decorators import require
-from sedona.utils.geometry_adapter import GeometryAdapter
+from sedona.spark.core.spatialOperator.rdd import SedonaRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.decorators import require
+from sedona.spark.utils.geometry_adapter import GeometryAdapter
class RangeQueryRaw:
diff --git a/python/sedona/core/spatialOperator/rdd.py b/python/sedona/spark/core/spatialOperator/rdd.py
similarity index 93%
rename from python/sedona/core/spatialOperator/rdd.py
rename to python/sedona/spark/core/spatialOperator/rdd.py
index 9576f731db..9f15fe416e 100644
--- a/python/sedona/core/spatialOperator/rdd.py
+++ b/python/sedona/spark/core/spatialOperator/rdd.py
@@ -17,8 +17,8 @@
from pyspark import RDD, SparkContext
-from sedona.core.jvm.translate import JvmSedonaPythonConverter
-from sedona.utils.spatial_rdd_parser import SedonaPickler
+from sedona.spark.core.jvm.translate import JvmSedonaPythonConverter
+from sedona.spark.utils.spatial_rdd_parser import SedonaPickler
class SedonaRDD:
diff --git a/python/sedona/core/utils.py b/python/sedona/spark/core/utils.py
similarity index 100%
rename from python/sedona/core/utils.py
rename to python/sedona/spark/core/utils.py
diff --git a/python/sedona/spark/geoarrow/__init__.py b/python/sedona/spark/geoarrow/__init__.py
new file mode 100644
index 0000000000..fb0c6f650e
--- /dev/null
+++ b/python/sedona/spark/geoarrow/__init__.py
@@ -0,0 +1,3 @@
+from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe
+
+__all__ = ["create_spatial_dataframe"]
diff --git a/python/sedona/utils/geoarrow.py b/python/sedona/spark/geoarrow/geoarrow.py
similarity index 99%
rename from python/sedona/utils/geoarrow.py
rename to python/sedona/spark/geoarrow/geoarrow.py
index b4a539dfa4..57d956eaf6 100644
--- a/python/sedona/utils/geoarrow.py
+++ b/python/sedona/spark/geoarrow/geoarrow.py
@@ -21,12 +21,12 @@ from typing import List, Callable
# with the ArrowStreamSerializer (instead of the ArrowCollectSerializer)
-from sedona.sql.st_functions import ST_AsEWKB
+from sedona.spark.sql.st_functions import ST_AsEWKB
from pyspark.sql import SparkSession
from pyspark.sql import DataFrame
from pyspark.sql.types import StructType, StructField, DataType, ArrayType, MapType
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
import geopandas as gpd
from pyspark.sql.pandas.types import (
from_arrow_type,
diff --git a/python/sedona/maps/SedonaKepler.py b/python/sedona/spark/maps/SedonaKepler.py
similarity index 95%
rename from python/sedona/maps/SedonaKepler.py
rename to python/sedona/spark/maps/SedonaKepler.py
index 6822cb28ad..b3536f4cb5 100644
--- a/python/sedona/maps/SedonaKepler.py
+++ b/python/sedona/spark/maps/SedonaKepler.py
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.maps.SedonaMapUtils import SedonaMapUtils
-from sedona.sql.types import RasterType
+from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils
+from sedona.spark.sql.types import RasterType
class SedonaKepler:
diff --git a/python/sedona/maps/SedonaMapUtils.py b/python/sedona/spark/maps/SedonaMapUtils.py
similarity index 97%
rename from python/sedona/maps/SedonaMapUtils.py
rename to python/sedona/spark/maps/SedonaMapUtils.py
index ddca721c45..21ada1b2aa 100644
--- a/python/sedona/maps/SedonaMapUtils.py
+++ b/python/sedona/spark/maps/SedonaMapUtils.py
@@ -17,8 +17,8 @@
import json
-from sedona.sql.types import GeometryType
-from sedona.utils.geoarrow import dataframe_to_arrow
+from sedona.spark.sql.types import GeometryType
+from sedona.spark.geoarrow.geoarrow import dataframe_to_arrow
from packaging.version import parse
diff --git a/python/sedona/maps/SedonaPyDeck.py b/python/sedona/spark/maps/SedonaPyDeck.py
similarity index 99%
rename from python/sedona/maps/SedonaPyDeck.py
rename to python/sedona/spark/maps/SedonaPyDeck.py
index a553817ec8..d2da836889 100644
--- a/python/sedona/maps/SedonaPyDeck.py
+++ b/python/sedona/spark/maps/SedonaPyDeck.py
@@ -27,7 +27,7 @@ from pyspark.sql.types import (
ShortType,
)
-from sedona.maps.SedonaMapUtils import SedonaMapUtils
+from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils
class SedonaPyDeck:
diff --git a/python/sedona/core/jvm/__init__.py b/python/sedona/spark/maps/__init__.py
similarity index 100%
rename from python/sedona/core/jvm/__init__.py
rename to python/sedona/spark/maps/__init__.py
diff --git a/python/sedona/maps/__init__.py b/python/sedona/spark/raster/__init__.py
similarity index 100%
rename from python/sedona/maps/__init__.py
rename to python/sedona/spark/raster/__init__.py
diff --git a/python/sedona/raster/awt_raster.py b/python/sedona/spark/raster/awt_raster.py
similarity index 100%
rename from python/sedona/raster/awt_raster.py
rename to python/sedona/spark/raster/awt_raster.py
diff --git a/python/sedona/raster/data_buffer.py b/python/sedona/spark/raster/data_buffer.py
similarity index 100%
rename from python/sedona/raster/data_buffer.py
rename to python/sedona/spark/raster/data_buffer.py
diff --git a/python/sedona/raster/meta.py b/python/sedona/spark/raster/meta.py
similarity index 100%
rename from python/sedona/raster/meta.py
rename to python/sedona/spark/raster/meta.py
diff --git a/python/sedona/raster/raster_serde.py b/python/sedona/spark/raster/raster_serde.py
similarity index 100%
rename from python/sedona/raster/raster_serde.py
rename to python/sedona/spark/raster/raster_serde.py
diff --git a/python/sedona/raster/sample_model.py b/python/sedona/spark/raster/sample_model.py
similarity index 100%
rename from python/sedona/raster/sample_model.py
rename to python/sedona/spark/raster/sample_model.py
diff --git a/python/sedona/raster/sedona_raster.py b/python/sedona/spark/raster/sedona_raster.py
similarity index 100%
rename from python/sedona/raster/sedona_raster.py
rename to python/sedona/spark/raster/sedona_raster.py
diff --git a/python/sedona/raster_utils/SedonaUtils.py b/python/sedona/spark/raster_utils/SedonaUtils.py
similarity index 94%
rename from python/sedona/raster_utils/SedonaUtils.py
rename to python/sedona/spark/raster_utils/SedonaUtils.py
index d35fcd6210..c28dbfb821 100644
--- a/python/sedona/raster_utils/SedonaUtils.py
+++ b/python/sedona/spark/raster_utils/SedonaUtils.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.maps.SedonaMapUtils import SedonaMapUtils
+from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils
class SedonaUtils:
diff --git a/python/sedona/raster/__init__.py b/python/sedona/spark/raster_utils/__init__.py
similarity index 100%
rename from python/sedona/raster/__init__.py
rename to python/sedona/spark/raster_utils/__init__.py
diff --git a/python/sedona/register/__init__.py b/python/sedona/spark/register/__init__.py
similarity index 92%
rename from python/sedona/register/__init__.py
rename to python/sedona/spark/register/__init__.py
index 9b6fcffe9d..b4f0f68202 100644
--- a/python/sedona/register/__init__.py
+++ b/python/sedona/spark/register/__init__.py
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.register.geo_registrator import SedonaRegistrator
+from sedona.spark.register.geo_registrator import SedonaRegistrator
__all__ = ["SedonaRegistrator"]
diff --git a/python/sedona/register/geo_registrator.py b/python/sedona/spark/register/geo_registrator.py
similarity index 91%
rename from python/sedona/register/geo_registrator.py
rename to python/sedona/spark/register/geo_registrator.py
index 8b80f3b17d..1b4de70a3e 100644
--- a/python/sedona/register/geo_registrator.py
+++ b/python/sedona/spark/register/geo_registrator.py
@@ -19,9 +19,9 @@ import attr
from py4j.java_gateway import java_import
from pyspark.sql import SparkSession
-from sedona.core.jvm.config import deprecated
-from sedona.register.java_libs import SedonaJvmLib
-from sedona.utils.prep import assign_all
+from sedona.spark.core.jvm.config import deprecated
+from sedona.spark.register.java_libs import SedonaJvmLib
+from sedona.spark.utils.prep import assign_all
assign_all()
jvm_import = str
@@ -55,7 +55,7 @@ class PackageImporter:
@staticmethod
def import_jvm_lib(jvm) -> bool:
- from sedona.core.utils import ImportedJvmLib
+ from sedona.spark.core.utils import ImportedJvmLib
"""
Imports all the specified methods and functions in jvm
diff --git a/python/sedona/register/java_libs.py b/python/sedona/spark/register/java_libs.py
similarity index 100%
rename from python/sedona/register/java_libs.py
rename to python/sedona/spark/register/java_libs.py
diff --git a/python/sedona/sql/__init__.py b/python/sedona/spark/sql/__init__.py
similarity index 74%
rename from python/sedona/sql/__init__.py
rename to python/sedona/spark/sql/__init__.py
index c8e002fb0a..3864d78f41 100644
--- a/python/sedona/sql/__init__.py
+++ b/python/sedona/spark/sql/__init__.py
@@ -14,20 +14,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import inspect
-import sys
# These allow use to access the __all__
-import sedona.sql.st_aggregates as st_aggregates
-import sedona.sql.st_constructors as st_constructors
-import sedona.sql.st_functions as st_functions
-import sedona.sql.st_predicates as st_predicates
+import sedona.spark.sql.st_aggregates as st_aggregates
+import sedona.spark.sql.st_constructors as st_constructors
+import sedona.spark.sql.st_functions as st_functions
+import sedona.spark.sql.st_predicates as st_predicates
# These bring the contents of the modules into this module
-from sedona.sql.st_aggregates import *
-from sedona.sql.st_constructors import *
-from sedona.sql.st_functions import *
-from sedona.sql.st_predicates import *
+from sedona.spark.sql.st_aggregates import *
+from sedona.spark.sql.st_constructors import *
+from sedona.spark.sql.st_functions import *
+from sedona.spark.sql.st_predicates import *
+import inspect
+import sys
+
__all__ = (
[
diff --git a/python/sedona/sql/connect.py b/python/sedona/spark/sql/connect.py
similarity index 100%
rename from python/sedona/sql/connect.py
rename to python/sedona/spark/sql/connect.py
diff --git a/python/sedona/sql/dataframe_api.py b/python/sedona/spark/sql/dataframe_api.py
similarity index 99%
rename from python/sedona/sql/dataframe_api.py
rename to python/sedona/spark/sql/dataframe_api.py
index 4c73a8af07..2efe479591 100644
--- a/python/sedona/sql/dataframe_api.py
+++ b/python/sedona/spark/sql/dataframe_api.py
@@ -37,7 +37,7 @@ except ImportError:
pass
else:
- from sedona.sql.connect import call_sedona_function_connect
+ from sedona.spark.sql.connect import call_sedona_function_connect
ColumnOrName = Union[Column, ConnectColumn, str]
diff --git a/python/sedona/sql/exceptions.py b/python/sedona/spark/sql/exceptions.py
similarity index 100%
rename from python/sedona/sql/exceptions.py
rename to python/sedona/spark/sql/exceptions.py
diff --git a/python/sedona/sql/functions.py b/python/sedona/spark/sql/functions.py
similarity index 97%
rename from python/sedona/sql/functions.py
rename to python/sedona/spark/sql/functions.py
index 83648ff53f..eb3b6f3c95 100644
--- a/python/sedona/sql/functions.py
+++ b/python/sedona/spark/sql/functions.py
@@ -21,8 +21,8 @@ from enum import Enum
import pandas as pd
-from sedona.sql.types import GeometryType
-from sedona.utils import geometry_serde
+from sedona.spark.sql.types import GeometryType
+from sedona.spark.utils import geometry_serde
from pyspark.sql.udf import UserDefinedFunction
from pyspark.sql.types import DataType
from shapely.geometry.base import BaseGeometry
diff --git a/python/sedona/sql/st_aggregates.py b/python/sedona/spark/sql/st_aggregates.py
similarity index 98%
rename from python/sedona/sql/st_aggregates.py
rename to python/sedona/spark/sql/st_aggregates.py
index 2156285bc2..f74cca7f84 100644
--- a/python/sedona/sql/st_aggregates.py
+++ b/python/sedona/spark/sql/st_aggregates.py
@@ -20,7 +20,7 @@ from functools import partial
from pyspark.sql import Column
-from sedona.sql.dataframe_api import (
+from sedona.spark.sql.dataframe_api import (
ColumnOrName,
call_sedona_function,
validate_argument_types,
diff --git a/python/sedona/sql/st_constructors.py b/python/sedona/spark/sql/st_constructors.py
similarity index 99%
rename from python/sedona/sql/st_constructors.py
rename to python/sedona/spark/sql/st_constructors.py
index c37f302af1..f47a33905d 100644
--- a/python/sedona/sql/st_constructors.py
+++ b/python/sedona/spark/sql/st_constructors.py
@@ -21,7 +21,7 @@ from typing import Optional, Union
from pyspark.sql import Column
-from sedona.sql.dataframe_api import (
+from sedona.spark.sql.dataframe_api import (
ColumnOrName,
ColumnOrNameOrNumber,
call_sedona_function,
diff --git a/python/sedona/sql/st_functions.py b/python/sedona/spark/sql/st_functions.py
similarity index 99%
rename from python/sedona/sql/st_functions.py
rename to python/sedona/spark/sql/st_functions.py
index 82bead1db2..495444917a 100644
--- a/python/sedona/sql/st_functions.py
+++ b/python/sedona/spark/sql/st_functions.py
@@ -22,7 +22,7 @@ from typing import Optional, Union
from pyspark.sql import Column
from pyspark.sql.functions import lit
-from sedona.sql.dataframe_api import (
+from sedona.spark.sql.dataframe_api import (
ColumnOrName,
ColumnOrNameOrNumber,
call_sedona_function,
diff --git a/python/sedona/sql/st_predicates.py b/python/sedona/spark/sql/st_predicates.py
similarity index 99%
rename from python/sedona/sql/st_predicates.py
rename to python/sedona/spark/sql/st_predicates.py
index 18b33c1605..71c464bffd 100644
--- a/python/sedona/sql/st_predicates.py
+++ b/python/sedona/spark/sql/st_predicates.py
@@ -21,7 +21,7 @@ from typing import Optional, Union
from pyspark.sql import Column
-from sedona.sql.dataframe_api import (
+from sedona.spark.sql.dataframe_api import (
ColumnOrName,
call_sedona_function,
validate_argument_types,
diff --git a/python/sedona/sql/types.py b/python/sedona/spark/sql/types.py
similarity index 89%
rename from python/sedona/sql/types.py
rename to python/sedona/spark/sql/types.py
index 1b8a9ee03b..5465b283eb 100644
--- a/python/sedona/sql/types.py
+++ b/python/sedona/spark/sql/types.py
@@ -24,16 +24,16 @@ except ImportError:
rasterio = None
if rasterio is not None:
- from ..raster import raster_serde
- from ..raster.sedona_raster import SedonaRaster
+ from sedona.spark.raster import raster_serde
+ from sedona.spark.raster.sedona_raster import SedonaRaster
else:
# We'll skip RasterType UDT registration and raise error when deserializing
# RasterUDT objects if rasterio is not installed
raster_serde = None
SedonaRaster = None
-from ..utils import geometry_serde
-from ..core.geom.geography import Geography
+from sedona.spark.utils import geometry_serde
+from sedona.spark.core.geom.geography import Geography
class GeometryType(UserDefinedType):
@@ -51,7 +51,7 @@ class GeometryType(UserDefinedType):
@classmethod
def module(cls):
- return "sedona.sql.types"
+ return "sedona.spark.sql.types"
def needConversion(self):
return True
@@ -76,7 +76,7 @@ class GeographyType(UserDefinedType):
@classmethod
def module(cls):
- return "sedona.sql.types"
+ return "sedona.spark.sql.types"
def needConversion(self):
return True
@@ -105,7 +105,7 @@ class RasterType(UserDefinedType):
@classmethod
def module(cls):
- return "sedona.sql.types"
+ return "sedona.spark.sql.types"
def needConversion(self):
return True
diff --git a/python/sedona/core/geom/shapely1/__init__.py b/python/sedona/spark/stac/__init__.py
similarity index 94%
rename from python/sedona/core/geom/shapely1/__init__.py
rename to python/sedona/spark/stac/__init__.py
index a67d5ea255..f4bd70986d 100644
--- a/python/sedona/core/geom/shapely1/__init__.py
+++ b/python/sedona/spark/stac/__init__.py
@@ -14,3 +14,4 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from sedona.spark.stac.client import Client
diff --git a/python/sedona/stac/client.py b/python/sedona/spark/stac/client.py
similarity index 98%
rename from python/sedona/stac/client.py
rename to python/sedona/spark/stac/client.py
index 50ce0afc6c..8f38cffc99 100644
--- a/python/sedona/stac/client.py
+++ b/python/sedona/spark/stac/client.py
@@ -16,7 +16,7 @@
# under the License.
from typing import Union, Optional, Iterator
-from sedona.stac.collection_client import CollectionClient
+from sedona.spark.stac.collection_client import CollectionClient
import datetime as python_datetime
from pystac import Item as PyStacItem
diff --git a/python/sedona/stac/collection_client.py b/python/sedona/spark/stac/collection_client.py
similarity index 100%
rename from python/sedona/stac/collection_client.py
rename to python/sedona/spark/stac/collection_client.py
diff --git a/python/tests/tools.py b/python/sedona/spark/stats/__init__.py
similarity index 64%
copy from python/tests/tools.py
copy to python/sedona/spark/stats/__init__.py
index 0e9be54aec..5219e02429 100644
--- a/python/tests/tools.py
+++ b/python/sedona/spark/stats/__init__.py
@@ -14,18 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from os import path
-
-from shapely.geometry import Point
-
-from sedona.utils.spatial_rdd_parser import GeoData
-
-tests_path = path.abspath(path.join(__file__, "../../../spark/common/src/test/"))
-tests_resource = path.abspath(
- path.join(__file__, "../../../spark/common/src/test/resources/")
+from sedona.spark.stats.clustering.dbscan import dbscan
+from sedona.spark.stats.outlier_detection.local_outlier_factor import (
+ local_outlier_factor,
)
-
-
-def distance_sorting_functions(geo_data: GeoData, query_point: Point):
- return geo_data.geom.distance(query_point)
+from sedona.spark.stats.hotspot_detection.getis_ord import g_local
+from sedona.spark.stats.weighting import add_distance_band_column
+from sedona.spark.stats.weighting import add_binary_distance_band_column
+from sedona.spark.stats.weighting import add_weighted_distance_band_column
diff --git a/python/sedona/stats/clustering/__init__.py b/python/sedona/spark/stats/clustering/__init__.py
similarity index 100%
copy from python/sedona/stats/clustering/__init__.py
copy to python/sedona/spark/stats/clustering/__init__.py
diff --git a/python/sedona/stats/clustering/dbscan.py b/python/sedona/spark/stats/clustering/dbscan.py
similarity index 100%
rename from python/sedona/stats/clustering/dbscan.py
rename to python/sedona/spark/stats/clustering/dbscan.py
diff --git a/python/sedona/stats/hotspot_detection/__init__.py b/python/sedona/spark/stats/hotspot_detection/__init__.py
similarity index 100%
copy from python/sedona/stats/hotspot_detection/__init__.py
copy to python/sedona/spark/stats/hotspot_detection/__init__.py
diff --git a/python/sedona/stats/hotspot_detection/getis_ord.py b/python/sedona/spark/stats/hotspot_detection/getis_ord.py
similarity index 100%
rename from python/sedona/stats/hotspot_detection/getis_ord.py
rename to python/sedona/spark/stats/hotspot_detection/getis_ord.py
diff --git a/python/sedona/stats/outlier_detection/__init__.py b/python/sedona/spark/stats/outlier_detection/__init__.py
similarity index 100%
copy from python/sedona/stats/outlier_detection/__init__.py
copy to python/sedona/spark/stats/outlier_detection/__init__.py
diff --git a/python/sedona/stats/outlier_detection/local_outlier_factor.py b/python/sedona/spark/stats/outlier_detection/local_outlier_factor.py
similarity index 100%
rename from python/sedona/stats/outlier_detection/local_outlier_factor.py
rename to python/sedona/spark/stats/outlier_detection/local_outlier_factor.py
diff --git a/python/sedona/stats/weighting.py b/python/sedona/spark/stats/weighting.py
similarity index 100%
rename from python/sedona/stats/weighting.py
rename to python/sedona/spark/stats/weighting.py
diff --git a/python/sedona/utils/__init__.py b/python/sedona/spark/utils/__init__.py
similarity index 100%
copy from python/sedona/utils/__init__.py
copy to python/sedona/spark/utils/__init__.py
diff --git a/python/sedona/utils/abstract_parser.py b/python/sedona/spark/utils/abstract_parser.py
similarity index 100%
rename from python/sedona/utils/abstract_parser.py
rename to python/sedona/spark/utils/abstract_parser.py
diff --git a/python/sedona/utils/adapter.py b/python/sedona/spark/utils/adapter.py
similarity index 94%
rename from python/sedona/utils/adapter.py
rename to python/sedona/spark/utils/adapter.py
index 7b978e7212..42db222070 100644
--- a/python/sedona/utils/adapter.py
+++ b/python/sedona/spark/utils/adapter.py
@@ -20,10 +20,10 @@ from typing import List
from pyspark import RDD
from pyspark.sql import DataFrame, SparkSession
-from sedona.core.enums.spatial import SpatialType
-from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.core.enums.spatial import SpatialType
+from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.utils.meta import MultipleMeta
class Adapter(metaclass=MultipleMeta):
@@ -40,7 +40,7 @@ class Adapter(metaclass=MultipleMeta):
@classmethod
def toRdd(cls, dataFrame: DataFrame) -> "JvmSpatialRDD":
- from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD
+ from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD
sc = dataFrame._sc
jvm = sc._jvm
diff --git a/python/sedona/utils/binary_parser.py b/python/sedona/spark/utils/binary_parser.py
similarity index 100%
rename from python/sedona/utils/binary_parser.py
rename to python/sedona/spark/utils/binary_parser.py
diff --git a/python/sedona/utils/decorators.py b/python/sedona/spark/utils/decorators.py
similarity index 97%
rename from python/sedona/utils/decorators.py
rename to python/sedona/spark/utils/decorators.py
index 2794baba18..50350bf6da 100644
--- a/python/sedona/utils/decorators.py
+++ b/python/sedona/spark/utils/decorators.py
@@ -44,7 +44,7 @@ def get_first_meet_criteria_element_from_iterable(
def require(library_names: List[str]):
def wrapper(func):
def run_function(*args, **kwargs):
- from sedona.core.utils import ImportedJvmLib
+ from sedona.spark.core.utils import ImportedJvmLib
has_all_libs = [lib for lib in library_names]
first_not_fulfill_value = get_first_meet_criteria_element_from_iterable(
diff --git a/python/sedona/utils/geometry_adapter.py b/python/sedona/spark/utils/geometry_adapter.py
similarity index 87%
rename from python/sedona/utils/geometry_adapter.py
rename to python/sedona/spark/utils/geometry_adapter.py
index 9a81307d86..00ca55e139 100644
--- a/python/sedona/utils/geometry_adapter.py
+++ b/python/sedona/spark/utils/geometry_adapter.py
@@ -17,9 +17,9 @@
from shapely.geometry.base import BaseGeometry
-from sedona.core.geom.envelope import Envelope
-from sedona.core.jvm.translate import JvmGeometryAdapter
-from sedona.utils.spatial_rdd_parser import GeometryFactory
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.jvm.translate import JvmGeometryAdapter
+from sedona.spark.utils.spatial_rdd_parser import GeometryFactory
class GeometryAdapter:
diff --git a/python/sedona/utils/geometry_serde.py b/python/sedona/spark/utils/geometry_serde.py
similarity index 100%
rename from python/sedona/utils/geometry_serde.py
rename to python/sedona/spark/utils/geometry_serde.py
diff --git a/python/sedona/utils/geometry_serde_general.py b/python/sedona/spark/utils/geometry_serde_general.py
similarity index 100%
rename from python/sedona/utils/geometry_serde_general.py
rename to python/sedona/spark/utils/geometry_serde_general.py
diff --git a/python/sedona/utils/jvm.py b/python/sedona/spark/utils/jvm.py
similarity index 92%
rename from python/sedona/utils/jvm.py
rename to python/sedona/spark/utils/jvm.py
index 1b843a001f..50b1670199 100644
--- a/python/sedona/utils/jvm.py
+++ b/python/sedona/spark/utils/jvm.py
@@ -17,8 +17,8 @@
import attr
-from sedona.core.jvm.abstract import JvmObject
-from sedona.utils.decorators import require
+from sedona.spark.core.jvm.abstract import JvmObject
+from sedona.spark.utils.decorators import require
@attr.s
diff --git a/python/sedona/utils/meta.py b/python/sedona/spark/utils/meta.py
similarity index 100%
rename from python/sedona/utils/meta.py
rename to python/sedona/spark/utils/meta.py
diff --git a/python/sedona/utils/prep.py b/python/sedona/spark/utils/prep.py
similarity index 91%
rename from python/sedona/utils/prep.py
rename to python/sedona/spark/utils/prep.py
index 8cabfa05c2..11bec77448 100644
--- a/python/sedona/utils/prep.py
+++ b/python/sedona/spark/utils/prep.py
@@ -28,7 +28,7 @@ from shapely.geometry import (
)
from shapely.geometry.base import BaseGeometry
-from ..core.geom.geography import Geography
+from sedona.spark.core.geom.geography import Geography
def assign_all() -> bool:
@@ -48,7 +48,7 @@ def assign_all() -> bool:
def assign_udt_shapely_objects(geoms: List[type(BaseGeometry)]) -> bool:
- from sedona.sql.types import GeometryType
+ from sedona.spark.sql.types import GeometryType
for geom in geoms:
geom.__UDT__ = GeometryType()
@@ -61,6 +61,6 @@ def assign_user_data_to_shapely_objects(geoms: List[type(BaseGeometry)]) -> bool
def assign_udt_geography():
- from sedona.sql.types import GeographyType
+ from sedona.spark.sql.types import GeographyType
Geography.__UDT__ = GeographyType()
diff --git a/python/sedona/utils/serde.py b/python/sedona/spark/utils/serde.py
similarity index 95%
rename from python/sedona/utils/serde.py
rename to python/sedona/spark/utils/serde.py
index e4d6e78684..37608c7b7b 100644
--- a/python/sedona/utils/serde.py
+++ b/python/sedona/spark/utils/serde.py
@@ -17,7 +17,7 @@
from abc import ABC
-from sedona.utils.decorators import classproperty
+from sedona.spark.utils.decorators import classproperty
class Serializer(ABC):
diff --git a/python/sedona/utils/spatial_rdd_parser.py b/python/sedona/spark/utils/spatial_rdd_parser.py
similarity index 96%
rename from python/sedona/utils/spatial_rdd_parser.py
rename to python/sedona/spark/utils/spatial_rdd_parser.py
index d99eecf807..0a8be26aca 100644
--- a/python/sedona/utils/spatial_rdd_parser.py
+++ b/python/sedona/spark/utils/spatial_rdd_parser.py
@@ -30,8 +30,7 @@ except ImportError:
from shapely.wkb import dumps
-from sedona.core.geom.circle import Circle
-from sedona.utils.binary_parser import BinaryParser
+from sedona.spark.utils.binary_parser import BinaryParser
class GeoData:
@@ -49,6 +48,8 @@ class GeoData:
return self.userData
def __getstate__(self):
+ from sedona.spark.core.geom.circle import Circle
+
attributes = copy(self.__slots__)
geom = getattr(self, attributes[0])
@@ -63,6 +64,8 @@ class GeoData:
)
def __setstate__(self, attributes):
+ from sedona.spark.core.geom.circle import Circle
+
bin_parser = BinaryParser(attributes["geom"])
is_circle = bin_parser.read_byte()
geom_bytes = attributes["geom"]
@@ -234,13 +237,15 @@ class CircleGeometryFactory:
@classmethod
def geometry_from_bytes(cls, bin_parser: BinaryParser) -> GeoData:
+ from sedona.spark.core.geom.circle import Circle
+
geom, user_data = read_geometry_from_bytes(bin_parser)
radius = bin_parser.read_double()
geo_data = GeoData(geom=Circle(geom, radius), userData=user_data)
return geo_data
@classmethod
- def to_bytes(cls, geom: Circle) -> List[int]:
+ def to_bytes(cls, geom: "Circle") -> List[int]:
return (
struct.pack("b", 1)
+ struct.pack("d", geom.radius)
diff --git a/python/sedona/utils/structured_adapter.py b/python/sedona/spark/utils/structured_adapter.py
similarity index 96%
rename from python/sedona/utils/structured_adapter.py
rename to python/sedona/spark/utils/structured_adapter.py
index bc3b959363..6af7df8a24 100644
--- a/python/sedona/utils/structured_adapter.py
+++ b/python/sedona/spark/utils/structured_adapter.py
@@ -18,8 +18,8 @@
from pyspark.sql import DataFrame, SparkSession
from pyspark.sql.types import StructType
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.core.spatialOperator.rdd import SedonaPairRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD
class StructuredAdapter:
diff --git a/python/sedona/utils/types.py b/python/sedona/spark/utils/types.py
similarity index 100%
rename from python/sedona/utils/types.py
rename to python/sedona/spark/utils/types.py
diff --git a/python/sedona/stac/__init__.py b/python/sedona/stac/__init__.py
index a67d5ea255..52cb22054b 100644
--- a/python/sedona/stac/__init__.py
+++ b/python/sedona/stac/__init__.py
@@ -1,16 +1,13 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+from sedona.spark.stac.client import Client
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.stac' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stac' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "Client",
+]
diff --git a/python/sedona/stats/__init__.py b/python/sedona/stats/__init__.py
index a67d5ea255..e69de29bb2 100644
--- a/python/sedona/stats/__init__.py
+++ b/python/sedona/stats/__init__.py
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/python/sedona/stats/clustering/__init__.py b/python/sedona/stats/clustering/__init__.py
index d2399abf8a..e69de29bb2 100644
--- a/python/sedona/stats/clustering/__init__.py
+++ b/python/sedona/stats/clustering/__init__.py
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""The clustering module contains spark based implementations of popular geospatial clustering algorithms.
-
-These implementations are designed to scale to larger datasets and support various geometric feature types.
-"""
diff --git a/python/sedona/stats/clustering/dbscan/__init__.py b/python/sedona/stats/clustering/dbscan/__init__.py
new file mode 100644
index 0000000000..a06bccdf64
--- /dev/null
+++ b/python/sedona/stats/clustering/dbscan/__init__.py
@@ -0,0 +1,11 @@
+from sedona.spark.stats.clustering.dbscan import dbscan
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.spark.stats.clustering.dbscan' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = ["dbscan"]
diff --git a/python/sedona/stats/hotspot_detection/__init__.py b/python/sedona/stats/hotspot_detection/__init__.py
index ed2c4706d1..e69de29bb2 100644
--- a/python/sedona/stats/hotspot_detection/__init__.py
+++ b/python/sedona/stats/hotspot_detection/__init__.py
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""Detecting across a region where a variable's value is significantly different from other values nearby."""
diff --git a/python/sedona/stats/hotspot_detection/getis_ord/__init__.py b/python/sedona/stats/hotspot_detection/getis_ord/__init__.py
new file mode 100644
index 0000000000..77b826b11d
--- /dev/null
+++ b/python/sedona/stats/hotspot_detection/getis_ord/__init__.py
@@ -0,0 +1,9 @@
+from sedona.spark.stats.hotspot_detection.getis_ord import g_local
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.stats.hotspot_detection.getis_ord' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
diff --git a/python/sedona/stats/outlier_detection/__init__.py b/python/sedona/stats/outlier_detection/__init__.py
index 4dd25a3ff6..e69de29bb2 100644
--- a/python/sedona/stats/outlier_detection/__init__.py
+++ b/python/sedona/stats/outlier_detection/__init__.py
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""Algorithms for detecting outliers in spatial datasets."""
diff --git a/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py b/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py
new file mode 100644
index 0000000000..3f3c614d5e
--- /dev/null
+++ b/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py
@@ -0,0 +1,13 @@
+from sedona.spark.stats.outlier_detection.local_outlier_factor import (
+ local_outlier_factor,
+)
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.stats.outlier_detection.local_outlier_factor' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = ["local_outlier_factor"]
diff --git a/python/sedona/stats/weighting/__init__.py b/python/sedona/stats/weighting/__init__.py
new file mode 100644
index 0000000000..1249261aa7
--- /dev/null
+++ b/python/sedona/stats/weighting/__init__.py
@@ -0,0 +1,9 @@
+from sedona.spark.stats.weighting import add_binary_distance_band_column
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.stats.weighting' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
diff --git a/python/sedona/utils/__init__.py b/python/sedona/utils/__init__.py
index 8c3ca09339..e69de29bb2 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/utils/__init__.py
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from .serde import KryoSerializer, SedonaKryoRegistrator
-
-__all__ = ["KryoSerializer", "SedonaKryoRegistrator"]
diff --git a/python/sedona/utils/geoarrow/__init__.py b/python/sedona/utils/geoarrow/__init__.py
new file mode 100644
index 0000000000..81e701d4ed
--- /dev/null
+++ b/python/sedona/utils/geoarrow/__init__.py
@@ -0,0 +1,11 @@
+import warnings
+
+from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe
+
+warnings.warn(
+ "The 'sedona.utils.geoarrow' module is deprecated and will be removed in future versions. Please use 'sedona.spark.geoarrow' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = ["create_spatial_dataframe"]
diff --git a/python/sedona/utils/structured_adapter/__init__.py b/python/sedona/utils/structured_adapter/__init__.py
new file mode 100644
index 0000000000..d500da2779
--- /dev/null
+++ b/python/sedona/utils/structured_adapter/__init__.py
@@ -0,0 +1,13 @@
+from sedona.spark.utils.structured_adapter import StructuredAdapter
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.utils.structured_adapter' module is deprecated and will be removed in future versions. Please use 'sedona.spark' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "StructuredAdapter",
+]
diff --git a/python/setup.py b/python/setup.py
index a6bc7e9560..695966631e 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -34,7 +34,7 @@ if os.getenv("ENABLE_ASAN"):
ext_modules = [
Extension(
- "sedona.utils.geomserde_speedup",
+ "sedona.spark.utils.geomserde_speedup",
sources=[
"src/geomserde_speedup_module.c",
"src/geomserde.c",
@@ -63,6 +63,7 @@ setup(
"spark": ["pyspark>=2.3.0"],
"pydeck-map": ["geopandas", "pydeck==0.8.0"],
"kepler-map": ["geopandas", "keplergl==0.3.2"],
+ "flink": ["apache-flink>=1.19.0"],
"all": [
"pyspark>=2.3.0",
"geopandas",
diff --git a/python/tests/__init__.py b/python/tests/__init__.py
index 0f8fa55be8..f05c0eb894 100644
--- a/python/tests/__init__.py
+++ b/python/tests/__init__.py
@@ -17,7 +17,12 @@
import os
-from tests.tools import tests_resource
+from os import path
+
+tests_path = os.path.abspath(os.path.join(__file__, "../../../spark/common/src/test/"))
+tests_resource = os.path.abspath(
+ path.join(__file__, "../../../spark/common/src/test/resources/")
+)
mixed_wkb_geometry_input_location = os.path.join(tests_resource, "county_small_wkb.tsv")
mixed_wkt_geometry_input_location = os.path.join(tests_resource, "county_small.tsv")
diff --git a/python/tests/core/test_avoiding_python_jvm_serde_df.py b/python/tests/core/test_avoiding_python_jvm_serde_df.py
index 117429d996..bb3df98863 100644
--- a/python/tests/core/test_avoiding_python_jvm_serde_df.py
+++ b/python/tests/core/test_avoiding_python_jvm_serde_df.py
@@ -18,17 +18,18 @@ import os
from pyspark.sql.types import StructField, StructType
from shapely.wkt import loads
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-
-from sedona.core.enums import GridType, IndexType
-from sedona.core.formatMapper import WktReader
-from sedona.core.spatialOperator.join_params import JoinParams
-from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw
-from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw
-from sedona.core.SpatialRDD import CircleRDD
-from sedona.sql.types import GeometryType
-from sedona.utils.adapter import Adapter
+
+from sedona.spark.core.enums import GridType, IndexType
+from sedona.spark.core.formatMapper import WktReader
+from sedona.spark.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw
+from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw
+from sedona.spark.core.SpatialRDD import CircleRDD
+from sedona.spark.sql.types import GeometryType
+from sedona.spark.utils.adapter import Adapter
bank_csv_path = os.path.join(tests_resource, "small/points.csv")
areas_csv_path = os.path.join(tests_resource, "small/areas.csv")
diff --git a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
index 5440034fd6..99c08ef2ff 100644
--- a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
+++ b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
@@ -18,15 +18,16 @@
import os
from shapely.wkt import loads
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-
-from sedona.core.enums import GridType, IndexType
-from sedona.core.formatMapper import WktReader
-from sedona.core.spatialOperator.join_params import JoinParams
-from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw
-from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw
-from sedona.core.SpatialRDD import CircleRDD
+
+from sedona.spark.core.enums import GridType, IndexType
+from sedona.spark.core.formatMapper import WktReader
+from sedona.spark.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw
+from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw
+from sedona.spark.core.SpatialRDD import CircleRDD
bank_csv_path = os.path.join(tests_resource, "small/points.csv")
areas_csv_path = os.path.join(tests_resource, "small/areas.csv")
diff --git a/python/tests/core/test_config.py b/python/tests/core/test_config.py
index f188d99e94..926a173283 100644
--- a/python/tests/core/test_config.py
+++ b/python/tests/core/test_config.py
@@ -19,7 +19,7 @@ import os
from tests.test_base import TestBase
-from sedona.core.jvm.config import SedonaMeta, SparkJars
+from sedona.spark.core.jvm.config import SedonaMeta, SparkJars
class TestCoreJVMConfig(TestBase):
diff --git a/python/tests/core/test_core_geom_primitives.py b/python/tests/core/test_core_geom_primitives.py
index 09717cef16..708b3f64c2 100644
--- a/python/tests/core/test_core_geom_primitives.py
+++ b/python/tests/core/test_core_geom_primitives.py
@@ -17,7 +17,7 @@
from tests.test_base import TestBase
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.geom.envelope import Envelope
class TestGeomPrimitives(TestBase):
diff --git a/python/tests/core/test_core_rdd.py b/python/tests/core/test_core_rdd.py
index 8567fd84d2..c83551b79b 100644
--- a/python/tests/core/test_core_rdd.py
+++ b/python/tests/core/test_core_rdd.py
@@ -17,11 +17,11 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter
-from sedona.core.SpatialRDD import PointRDD, PolygonRDD
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.SpatialRDD import PointRDD, PolygonRDD
point_path = os.path.join(tests_resource, "points.csv")
counties_path = os.path.join(tests_resource, "county_small.tsv")
diff --git a/python/tests/core/test_rdd.py b/python/tests/core/test_rdd.py
index abf5e1c9b9..ff649a58f9 100644
--- a/python/tests/core/test_rdd.py
+++ b/python/tests/core/test_rdd.py
@@ -19,6 +19,8 @@ import logging
import os
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.properties.polygon_properties import (
polygon_rdd_end_offset,
polygon_rdd_index_type,
@@ -27,14 +29,13 @@ from tests.properties.polygon_properties import (
polygon_rdd_start_offset,
)
from tests.test_base import TestBase
-from tests.tools import tests_resource
-
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery
-from sedona.core.spatialOperator.join_params import JoinParams
-from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD
+
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery
+from sedona.spark.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD
point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv")
diff --git a/python/tests/core/test_spatial_rdd_from_disc.py b/python/tests/core/test_spatial_rdd_from_disc.py
index f3f3cbabf6..74cee595cf 100644
--- a/python/tests/core/test_spatial_rdd_from_disc.py
+++ b/python/tests/core/test_spatial_rdd_from_disc.py
@@ -18,18 +18,17 @@
import os
import shutil
-import pytest
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import GridType, IndexType
-from sedona.core.formatMapper.disc_utils import (
+from sedona.spark.core.enums import GridType, IndexType
+from sedona.spark.core.formatMapper.disc_utils import (
GeoType,
load_spatial_index_rdd_from_disc,
load_spatial_rdd_from_disc,
)
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
def remove_directory(path: str) -> bool:
diff --git a/python/tests/flink/conftest.py b/python/tests/flink/conftest.py
new file mode 100644
index 0000000000..2163f3c514
--- /dev/null
+++ b/python/tests/flink/conftest.py
@@ -0,0 +1,43 @@
+import os
+
+import pytest
+
+
+EXTRA_JARS = os.getenv("SEDONA_PYFLINK_EXTRA_JARS")
+
+
+def has_pyflink():
+ try:
+ import pyflink
+ except ImportError:
+ return False
+ return True
+
+
+if has_pyflink():
+ from sedona.flink import SedonaContext
+
+ try:
+ from pyflink.datastream import StreamExecutionEnvironment
+ from pyflink.table import EnvironmentSettings, StreamTableEnvironment
+ except ImportError:
+ pytest.skip("PyFlink is not installed. Skipping tests that require PyFlink.")
+
+ @pytest.fixture(scope="module")
+ def flink_settings():
+ return EnvironmentSettings.in_streaming_mode()
+
+ @pytest.fixture(scope="module")
+ def stream_env() -> StreamExecutionEnvironment:
+ env = StreamExecutionEnvironment.get_execution_environment()
+ jars = EXTRA_JARS.split(",") if EXTRA_JARS else []
+ for jar in jars:
+ env.add_jars(f"file://{jar}")
+
+ return env
+
+ @pytest.fixture(scope="module")
+ def table_env(
+ stream_env: StreamExecutionEnvironment, flink_settings: EnvironmentSettings
+ ) -> StreamTableEnvironment:
+ return SedonaContext.create(stream_env, flink_settings)
diff --git a/python/tests/flink/test_flink_registration.py b/python/tests/flink/test_flink_registration.py
new file mode 100644
index 0000000000..02c6c08261
--- /dev/null
+++ b/python/tests/flink/test_flink_registration.py
@@ -0,0 +1,45 @@
+from shapely.wkb import loads
+import pytest
+
+
+@pytest.mark.flink
+def test_register(table_env):
+ result = (
+ table_env.sql_query("SELECT ST_ASBinary(ST_Point(1.0, 2.0))")
+ .execute()
+ .collect()
+ )
+
+ assert 1 == len([el for el in result])
+
+
+@pytest.mark.flink
+def test_register_udf(table_env):
+ from pyflink.table.udf import ScalarFunction, udf
+
+ class Buffer(ScalarFunction):
+ def eval(self, s):
+ geom = loads(s)
+ return geom.buffer(1).wkb
+
+ table_env.create_temporary_function(
+ "ST_BufferPython", udf(Buffer(), result_type="Binary")
+ )
+
+ buffer_table = table_env.sql_query(
+ "SELECT ST_BufferPython(ST_ASBinary(ST_Point(1.0, 2.0))) AS buffer"
+ )
+
+ table_env.create_temporary_view("buffer_table", buffer_table)
+
+ result = (
+ table_env.sql_query("SELECT ST_Area(ST_GeomFromWKB(buffer)) FROM buffer_table")
+ .execute()
+ .collect()
+ )
+
+ items = [el for el in result]
+ area = items[0][0]
+
+ assert 3.12 < area < 3.14
+ assert 1 == len(items)
diff --git a/python/tests/format_mapper/test_geo_json_reader.py b/python/tests/format_mapper/test_geo_json_reader.py
index 042bd08fce..5b550ea51d 100644
--- a/python/tests/format_mapper/test_geo_json_reader.py
+++ b/python/tests/format_mapper/test_geo_json_reader.py
@@ -17,12 +17,11 @@
import os
-import pyspark
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.formatMapper.geo_json_reader import GeoJsonReader
-from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version
+from sedona.spark.core.formatMapper.geo_json_reader import GeoJsonReader
+from sedona.spark.core.jvm.config import SedonaMeta, is_greater_or_equal_version
geo_json_contains_id = os.path.join(tests_resource, "testContainsId.json")
geo_json_geom_with_feature_property = os.path.join(tests_resource, "testPolygon.json")
diff --git a/python/tests/format_mapper/test_shapefile_reader.py b/python/tests/format_mapper/test_shapefile_reader.py
index 8fe47154a9..0a8afd1f9b 100644
--- a/python/tests/format_mapper/test_shapefile_reader.py
+++ b/python/tests/format_mapper/test_shapefile_reader.py
@@ -18,13 +18,13 @@
import os
import pytest
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.formatMapper.shapefileParser import ShapefileReader
-from sedona.core.geom.envelope import Envelope
-from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version
-from sedona.core.spatialOperator import RangeQuery
+from sedona.spark.core.formatMapper.shapefileParser import ShapefileReader
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQuery
undefined_type_shape_location = os.path.join(tests_resource, "shapefiles/undefined")
polygon_shape_location = os.path.join(tests_resource, "shapefiles/polygon")
diff --git a/python/tests/format_mapper/test_wkb_reader.py b/python/tests/format_mapper/test_wkb_reader.py
index d9452babbc..c3dc5df531 100644
--- a/python/tests/format_mapper/test_wkb_reader.py
+++ b/python/tests/format_mapper/test_wkb_reader.py
@@ -17,10 +17,10 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.formatMapper.wkb_reader import WkbReader
+from sedona.spark.core.formatMapper.wkb_reader import WkbReader
class TestWkbReader(TestBase):
diff --git a/python/tests/format_mapper/test_wkt_reader.py b/python/tests/format_mapper/test_wkt_reader.py
index ec64158fe3..83e7715ff0 100644
--- a/python/tests/format_mapper/test_wkt_reader.py
+++ b/python/tests/format_mapper/test_wkt_reader.py
@@ -17,10 +17,10 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.formatMapper import WktReader
+from sedona.spark.core.formatMapper import WktReader
class TestWktReader(TestBase):
diff --git a/python/tests/maps/test_sedonakepler_visualization.py b/python/tests/maps/test_sedonakepler_visualization.py
index 808b72a2f1..24501aa06d 100644
--- a/python/tests/maps/test_sedonakepler_visualization.py
+++ b/python/tests/maps/test_sedonakepler_visualization.py
@@ -25,7 +25,7 @@ from tests import (
)
from tests.test_base import TestBase
-from sedona.maps.SedonaKepler import SedonaKepler
+from sedona.spark import SedonaKepler
class TestVisualization(TestBase):
diff --git a/python/tests/maps/test_sedonapydeck.py b/python/tests/maps/test_sedonapydeck.py
index daa1d397b4..106fc28eb2 100644
--- a/python/tests/maps/test_sedonapydeck.py
+++ b/python/tests/maps/test_sedonapydeck.py
@@ -22,7 +22,7 @@ import pydeck as pdk
from tests import chicago_crimes_input_location, google_buildings_input_location
from tests.test_base import TestBase
-from sedona.maps.SedonaPyDeck import SedonaPyDeck
+from sedona.spark import SedonaPyDeck
class TestVisualization(TestBase):
diff --git a/python/tests/properties/crs_transform.py b/python/tests/properties/crs_transform.py
index af6437a0ef..0ae05c143b 100644
--- a/python/tests/properties/crs_transform.py
+++ b/python/tests/properties/crs_transform.py
@@ -18,10 +18,10 @@
import os
from shapely.geometry import Point
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from tests import tests_resource
input_location = os.path.join(tests_resource, "crs-test-point.csv")
offset = 0
diff --git a/python/tests/properties/linestring_properties.py b/python/tests/properties/linestring_properties.py
index e53fc14b72..627197180d 100644
--- a/python/tests/properties/linestring_properties.py
+++ b/python/tests/properties/linestring_properties.py
@@ -17,10 +17,10 @@
import os
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.geom.envelope import Envelope
+from tests import tests_resource
input_location = os.path.join(tests_resource, "primaryroads-linestring.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/properties/point_properties.py b/python/tests/properties/point_properties.py
index bea7dcf105..664b2bfb34 100644
--- a/python/tests/properties/point_properties.py
+++ b/python/tests/properties/point_properties.py
@@ -17,10 +17,9 @@
import os
-from tests.tools import tests_resource
-
-from sedona.core.enums import FileDataSplitter
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.geom.envelope import Envelope
+from tests import tests_resource
input_location = os.path.join(tests_resource, "arealm-small.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/properties/polygon_properties.py b/python/tests/properties/polygon_properties.py
index 09680f0861..6e1912836c 100644
--- a/python/tests/properties/polygon_properties.py
+++ b/python/tests/properties/polygon_properties.py
@@ -17,10 +17,9 @@
import os
-from tests.tools import tests_resource
-
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from tests import tests_resource
input_location = os.path.join(tests_resource, "primaryroads-polygon.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/raster/test_meta.py b/python/tests/raster/test_meta.py
index 9c4d01dd59..903b101ff8 100644
--- a/python/tests/raster/test_meta.py
+++ b/python/tests/raster/test_meta.py
@@ -15,10 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-import pytest
from pytest import approx
-from sedona.raster.meta import AffineTransform, PixelAnchor
+from sedona.spark.raster.meta import AffineTransform, PixelAnchor
class TestAffineTransform:
diff --git a/python/tests/raster/test_pandas_udf.py b/python/tests/raster/test_pandas_udf.py
index b33c8370d9..31bf210ebe 100644
--- a/python/tests/raster/test_pandas_udf.py
+++ b/python/tests/raster/test_pandas_udf.py
@@ -22,7 +22,6 @@ import pytest
import rasterio
from pyspark.sql.functions import expr, pandas_udf
from pyspark.sql.types import IntegerType
-from tests import world_map_raster_input_location
from tests.test_base import TestBase
@@ -42,7 +41,7 @@ class TestRasterPandasUDF(TestBase):
# A Python Pandas UDF that takes a raster as input
@pandas_udf(IntegerType())
def pandas_udf_raster_as_param(s: pd.Series) -> pd.Series:
- from sedona.raster import raster_serde
+ from sedona.spark.raster import raster_serde
def func(x):
with raster_serde.deserialize(x) as raster:
@@ -54,7 +53,7 @@ class TestRasterPandasUDF(TestBase):
# A Python Pandas UDF that takes a raster as input
@pandas_udf(IntegerType())
def pandas_udf_raster_as_param_2(s: pd.Series) -> pd.Series:
- from sedona.raster import raster_serde
+ from sedona.spark.raster import raster_serde
def func(x):
with raster_serde.deserialize(x) as raster:
diff --git a/python/tests/raster/test_serde.py b/python/tests/raster/test_serde.py
index 4dfe8b973a..05c0c0e1ac 100644
--- a/python/tests/raster/test_serde.py
+++ b/python/tests/raster/test_serde.py
@@ -22,7 +22,7 @@ from pyspark.sql.functions import expr
from tests import world_map_raster_input_location
from tests.test_base import TestBase
-from sedona.sql.types import RasterType
+from sedona.spark.sql.types import RasterType
class TestRasterSerde(TestBase):
diff --git a/python/tests/raster_viz_utils/test_sedonautils.py b/python/tests/raster_viz_utils/test_sedonautils.py
index 28f3f91b19..e3202d010e 100644
--- a/python/tests/raster_viz_utils/test_sedonautils.py
+++ b/python/tests/raster_viz_utils/test_sedonautils.py
@@ -15,11 +15,10 @@
# specific language governing permissions and limitations
# under the License.
-from IPython.display import HTML, display
from tests import world_map_raster_input_location
from tests.test_base import TestBase
-from sedona.raster_utils.SedonaUtils import SedonaUtils
+from sedona.spark import SedonaUtils
class TestSedonaUtils(TestBase):
diff --git a/python/tests/serialization/test_direct_serialization.py b/python/tests/serialization/test_direct_serialization.py
index c960468c87..fa87268b66 100644
--- a/python/tests/serialization/test_direct_serialization.py
+++ b/python/tests/serialization/test_direct_serialization.py
@@ -19,7 +19,7 @@ from shapely.geometry import Polygon
from shapely.wkt import loads
from tests.test_base import TestBase
-from sedona.utils.geometry_adapter import GeometryAdapter
+from sedona.spark.utils.geometry_adapter import GeometryAdapter
class TestDirectSerialization(TestBase):
diff --git a/python/tests/serialization/test_rdd_serialization.py b/python/tests/serialization/test_rdd_serialization.py
index e82ae1ce8e..636f1fa697 100644
--- a/python/tests/serialization/test_rdd_serialization.py
+++ b/python/tests/serialization/test_rdd_serialization.py
@@ -17,11 +17,11 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv")
polygon_rdd_input_location = os.path.join(tests_resource, "primaryroads-polygon.csv")
diff --git a/python/tests/serialization/test_serializers.py b/python/tests/serialization/test_serializers.py
index 362462c6ee..1e05dfaee6 100644
--- a/python/tests/serialization/test_serializers.py
+++ b/python/tests/serialization/test_serializers.py
@@ -32,7 +32,7 @@ from shapely.geometry import (
from tests import tests_resource
from tests.test_base import TestBase
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
class TestsSerializers(TestBase):
diff --git a/python/tests/serialization/test_with_sc_parellize.py b/python/tests/serialization/test_with_sc_parellize.py
index 60852704c2..138e905859 100644
--- a/python/tests/serialization/test_with_sc_parellize.py
+++ b/python/tests/serialization/test_with_sc_parellize.py
@@ -17,10 +17,11 @@
from shapely.geometry import LineString, Point, Polygon
from shapely.wkt import loads
+
+from sedona.spark.utils.spatial_rdd_parser import GeoData
from tests.test_base import TestBase
-from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
-from sedona.utils.spatial_rdd_parser import GeoData
+from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD
class TestWithScParallelize(TestBase):
diff --git a/python/tests/spatial_operator/test_join_base.py b/python/tests/spatial_operator/test_join_base.py
index 2aac30fb52..b6f19e2da4 100644
--- a/python/tests/spatial_operator/test_join_base.py
+++ b/python/tests/spatial_operator/test_join_base.py
@@ -15,12 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-import pytest
from tests.test_base import TestBase
-from sedona.core.enums import GridType
-from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, RectangleRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.core.enums import GridType
+from sedona.spark.core.SpatialRDD import (
+ LineStringRDD,
+ PointRDD,
+ PolygonRDD,
+ RectangleRDD,
+)
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
class TestJoinBase(TestBase):
diff --git a/python/tests/spatial_operator/test_join_query_correctness.py b/python/tests/spatial_operator/test_join_query_correctness.py
index e4995bef44..1659bc9393 100644
--- a/python/tests/spatial_operator/test_join_query_correctness.py
+++ b/python/tests/spatial_operator/test_join_query_correctness.py
@@ -17,13 +17,14 @@
from shapely.geometry import LineString, Point, Polygon
from shapely.geometry.base import BaseGeometry
+
+from sedona.spark.utils.spatial_rdd_parser import GeoData
from tests.test_base import TestBase
-from sedona.core.enums import GridType, IndexType
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.utils.spatial_rdd_parser import GeoData
+from sedona.spark.core.enums import GridType, IndexType
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
class TestJoinQueryCorrectness(TestBase):
diff --git a/python/tests/spatial_operator/test_linestring_join.py b/python/tests/spatial_operator/test_linestring_join.py
index af140d4118..b40d5247e2 100644
--- a/python/tests/spatial_operator/test_linestring_join.py
+++ b/python/tests/spatial_operator/test_linestring_join.py
@@ -17,14 +17,13 @@
import os
-import pytest
+from tests import tests_resource
from tests.spatial_operator.test_join_base import TestJoinBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.spatialOperator.join_params import JoinParams
input_location = os.path.join(tests_resource, "primaryroads-linestring.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_linestring_knn.py b/python/tests/spatial_operator/test_linestring_knn.py
index 8de3ab93c7..f5a9af29da 100644
--- a/python/tests/spatial_operator/test_linestring_knn.py
+++ b/python/tests/spatial_operator/test_linestring_knn.py
@@ -18,12 +18,13 @@
import os
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.spatialOperator import KNNQuery
-from sedona.core.SpatialRDD import LineStringRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.spatialOperator import KNNQuery
+from sedona.spark.core.SpatialRDD import LineStringRDD
input_location = os.path.join(tests_resource, "primaryroads-linestring.csv")
offset = 0
diff --git a/python/tests/spatial_operator/test_linestring_range.py b/python/tests/spatial_operator/test_linestring_range.py
index e6b5df971a..0f50785f6b 100644
--- a/python/tests/spatial_operator/test_linestring_range.py
+++ b/python/tests/spatial_operator/test_linestring_range.py
@@ -17,13 +17,13 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import RangeQuery
-from sedona.core.SpatialRDD import LineStringRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQuery
+from sedona.spark.core.SpatialRDD import LineStringRDD
input_location = os.path.join(tests_resource, "primaryroads-linestring.csv")
offset = 0
diff --git a/python/tests/spatial_operator/test_point_join.py b/python/tests/spatial_operator/test_point_join.py
index dcfdaeb790..9943f53265 100644
--- a/python/tests/spatial_operator/test_point_join.py
+++ b/python/tests/spatial_operator/test_point_join.py
@@ -17,14 +17,13 @@
import os
-import pytest
+from tests import tests_resource
from tests.spatial_operator.test_join_base import TestJoinBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.spatialOperator.join_params import JoinParams
input_location = os.path.join(tests_resource, "arealm-small.csv")
input_location_query_window = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_point_knn.py b/python/tests/spatial_operator/test_point_knn.py
index 5f5cc951f9..eff101b53a 100644
--- a/python/tests/spatial_operator/test_point_knn.py
+++ b/python/tests/spatial_operator/test_point_knn.py
@@ -18,12 +18,14 @@
import os
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import distance_sorting_functions, tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.spatialOperator import KNNQuery
-from sedona.core.SpatialRDD import PointRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.spatialOperator import KNNQuery
+from sedona.spark.core.SpatialRDD import PointRDD
+from tests.tools import distance_sorting_functions
input_location = os.path.join(tests_resource, "arealm-small.csv")
queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_point_range.py b/python/tests/spatial_operator/test_point_range.py
index 0c85af2359..d9fd788e8d 100644
--- a/python/tests/spatial_operator/test_point_range.py
+++ b/python/tests/spatial_operator/test_point_range.py
@@ -17,13 +17,13 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import RangeQuery
-from sedona.core.SpatialRDD import PointRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQuery
+from sedona.spark.core.SpatialRDD import PointRDD
input_location = os.path.join(tests_resource, "arealm-small.csv")
queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_polygon_join.py b/python/tests/spatial_operator/test_polygon_join.py
index 70e55e9659..2f4a6688aa 100644
--- a/python/tests/spatial_operator/test_polygon_join.py
+++ b/python/tests/spatial_operator/test_polygon_join.py
@@ -17,14 +17,13 @@
import os
-import pytest
+from tests import tests_resource
from tests.spatial_operator.test_join_base import TestJoinBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.spatialOperator.join_params import JoinParams
input_location = os.path.join(tests_resource, "primaryroads-polygon.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_polygon_knn.py b/python/tests/spatial_operator/test_polygon_knn.py
index fd604036c3..4da876a556 100644
--- a/python/tests/spatial_operator/test_polygon_knn.py
+++ b/python/tests/spatial_operator/test_polygon_knn.py
@@ -18,12 +18,14 @@
import os
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import distance_sorting_functions, tests_resource
+from tests.tools import distance_sorting_functions
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.spatialOperator import KNNQuery
-from sedona.core.SpatialRDD import PolygonRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.spatialOperator import KNNQuery
+from sedona.spark.core.SpatialRDD import PolygonRDD
input_location = os.path.join(tests_resource, "primaryroads-polygon.csv")
splitter = FileDataSplitter.CSV
diff --git a/python/tests/spatial_operator/test_polygon_range.py b/python/tests/spatial_operator/test_polygon_range.py
index cef018ab45..f2e359f433 100644
--- a/python/tests/spatial_operator/test_polygon_range.py
+++ b/python/tests/spatial_operator/test_polygon_range.py
@@ -17,13 +17,13 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import RangeQuery
-from sedona.core.SpatialRDD import PolygonRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQuery
+from sedona.spark.core.SpatialRDD import PolygonRDD
input_location = os.path.join(tests_resource, "primaryroads-polygon.csv")
splitter = FileDataSplitter.CSV
diff --git a/python/tests/spatial_operator/test_rectangle_join.py b/python/tests/spatial_operator/test_rectangle_join.py
index fc69f8d1ce..eea91796da 100644
--- a/python/tests/spatial_operator/test_rectangle_join.py
+++ b/python/tests/spatial_operator/test_rectangle_join.py
@@ -17,14 +17,14 @@
import os
+from tests import tests_resource
from tests.spatial_operator.test_join_base import TestJoinBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.enums.join_build_side import JoinBuildSide
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.spatialOperator.join_params import JoinParams
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.enums.join_build_side import JoinBuildSide
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.spatialOperator.join_params import JoinParams
input_location = os.path.join(tests_resource, "zcta510-small.csv")
query_window_set = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_rectangle_knn.py b/python/tests/spatial_operator/test_rectangle_knn.py
index 793f30f20d..30a6159d78 100644
--- a/python/tests/spatial_operator/test_rectangle_knn.py
+++ b/python/tests/spatial_operator/test_rectangle_knn.py
@@ -18,13 +18,15 @@
import os
from shapely.geometry import LineString, Point, Polygon
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import distance_sorting_functions, tests_resource
+from tests.tools import distance_sorting_functions
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import KNNQuery
-from sedona.core.SpatialRDD import RectangleRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import KNNQuery
+from sedona.spark.core.SpatialRDD import RectangleRDD
inputLocation = os.path.join(tests_resource, "zcta510-small.csv")
queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_operator/test_rectangle_range.py b/python/tests/spatial_operator/test_rectangle_range.py
index 8dbde75434..39a75bbd45 100644
--- a/python/tests/spatial_operator/test_rectangle_range.py
+++ b/python/tests/spatial_operator/test_rectangle_range.py
@@ -17,13 +17,13 @@
import os
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import RangeQuery
-from sedona.core.SpatialRDD import RectangleRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import RangeQuery
+from sedona.spark.core.SpatialRDD import RectangleRDD
inputLocation = os.path.join(tests_resource, "zcta510-small.csv")
queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_rdd/test_circle_rdd.py b/python/tests/spatial_rdd/test_circle_rdd.py
index d0151ec880..56cd54f1aa 100644
--- a/python/tests/spatial_rdd/test_circle_rdd.py
+++ b/python/tests/spatial_rdd/test_circle_rdd.py
@@ -23,7 +23,7 @@ from tests.properties.point_properties import (
)
from tests.test_base import TestBase
-from sedona.core.SpatialRDD import CircleRDD, PointRDD
+from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD
class TestCircleRDD(TestBase):
diff --git a/python/tests/spatial_rdd/test_linestring_rdd.py b/python/tests/spatial_rdd/test_linestring_rdd.py
index 04ff76a09a..4c1746c3d7 100644
--- a/python/tests/spatial_rdd/test_linestring_rdd.py
+++ b/python/tests/spatial_rdd/test_linestring_rdd.py
@@ -23,14 +23,12 @@ from tests.properties.linestring_properties import (
input_location,
num_partitions,
splitter,
- transformed_envelope,
- transformed_envelope_2,
)
from tests.test_base import TestBase
-from sedona.core.enums import GridType, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import LineStringRDD
+from sedona.spark.core.enums import IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import LineStringRDD
class TestLineStringRDD(TestBase):
diff --git a/python/tests/spatial_rdd/test_point_rdd.py b/python/tests/spatial_rdd/test_point_rdd.py
index e21bbf4c52..0335c9e5ab 100644
--- a/python/tests/spatial_rdd/test_point_rdd.py
+++ b/python/tests/spatial_rdd/test_point_rdd.py
@@ -17,7 +17,6 @@
from tests.properties.point_properties import (
crs_envelope,
- crs_envelope_transformed,
crs_point_test,
input_boundary,
input_count,
@@ -25,14 +24,13 @@ from tests.properties.point_properties import (
num_partitions,
offset,
splitter,
- transformed_envelope,
)
from tests.test_base import TestBase
-from sedona.core.enums import GridType, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import PointRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.core.enums import GridType, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import PointRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
class TestPointRDD(TestBase):
diff --git a/python/tests/spatial_rdd/test_polygon_rdd.py b/python/tests/spatial_rdd/test_polygon_rdd.py
index a038f84ab4..b030bfec56 100644
--- a/python/tests/spatial_rdd/test_polygon_rdd.py
+++ b/python/tests/spatial_rdd/test_polygon_rdd.py
@@ -28,15 +28,14 @@ from tests.properties.polygon_properties import (
polygon_rdd_input_location,
polygon_rdd_splitter,
polygon_rdd_start_offset,
- query_envelope,
splitter,
)
from tests.test_base import TestBase
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import PolygonRDD
-from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import PolygonRDD
+from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD
class TestPolygonRDD(TestBase):
diff --git a/python/tests/spatial_rdd/test_rectangle_rdd.py b/python/tests/spatial_rdd/test_rectangle_rdd.py
index 4c67e751ae..c964101165 100644
--- a/python/tests/spatial_rdd/test_rectangle_rdd.py
+++ b/python/tests/spatial_rdd/test_rectangle_rdd.py
@@ -17,13 +17,13 @@
import os
-import pytest
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import RectangleRDD
+from sedona.spark.core.enums import FileDataSplitter, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import RectangleRDD
+
inputLocation = os.path.join(tests_resource, "zcta510-small.csv")
queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv")
diff --git a/python/tests/spatial_rdd/test_spatial_rdd.py b/python/tests/spatial_rdd/test_spatial_rdd.py
index 4230b87c53..f68c429387 100644
--- a/python/tests/spatial_rdd/test_spatial_rdd.py
+++ b/python/tests/spatial_rdd/test_spatial_rdd.py
@@ -17,18 +17,18 @@
import os
-import pyspark
import pytest
from pyspark import RDD
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.formatMapper.geo_json_reader import GeoJsonReader
-from sedona.utils.adapter import Adapter
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import PointRDD
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.formatMapper.geo_json_reader import GeoJsonReader
+from sedona.spark import Adapter
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import PointRDD
input_file_location = os.path.join(tests_resource, "arealm-small.csv")
crs_test_point = os.path.join(tests_resource, "crs-test-point.csv")
diff --git a/python/tests/spatial_rdd/test_spatial_rdd_writer.py b/python/tests/spatial_rdd/test_spatial_rdd_writer.py
index 2e5a571c36..b01b126545 100644
--- a/python/tests/spatial_rdd/test_spatial_rdd_writer.py
+++ b/python/tests/spatial_rdd/test_spatial_rdd_writer.py
@@ -19,12 +19,13 @@ import os
import shutil
import pytest
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter
-from sedona.core.geom.envelope import Envelope
-from sedona.core.SpatialRDD import PointRDD
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.SpatialRDD import PointRDD
wkb_folder = "wkb"
wkt_folder = "wkt"
diff --git a/python/tests/sql/resource/sample_data.py b/python/tests/sql/resource/sample_data.py
index d907f66c48..7c3b27fa6f 100644
--- a/python/tests/sql/resource/sample_data.py
+++ b/python/tests/sql/resource/sample_data.py
@@ -23,7 +23,7 @@ from pyspark.sql import DataFrame, SparkSession
from pyspark.sql.types import StructField, StructType
from shapely import wkt
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
data_path = path.abspath(path.dirname(__file__))
diff --git a/python/tests/sql/test_adapter.py b/python/tests/sql/test_adapter.py
index 1c8ff34121..396d7948a3 100644
--- a/python/tests/sql/test_adapter.py
+++ b/python/tests/sql/test_adapter.py
@@ -17,7 +17,6 @@
import logging
-import pyspark
import pytest
from pyspark.sql import DataFrame
from pyspark.sql.functions import col, expr
@@ -32,13 +31,15 @@ from tests import (
from tests.test_base import TestBase
from sedona import version
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.formatMapper.shapefileParser.shape_file_reader import ShapefileReader
-from sedona.core.geom.envelope import Envelope
-from sedona.core.jvm.config import is_greater_or_equal_version
-from sedona.core.spatialOperator import JoinQuery
-from sedona.core.SpatialRDD import CircleRDD, PolygonRDD
-from sedona.utils.adapter import Adapter
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.formatMapper.shapefileParser.shape_file_reader import (
+ ShapefileReader,
+)
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.jvm.config import is_greater_or_equal_version
+from sedona.spark.core.spatialOperator import JoinQuery
+from sedona.spark.core.SpatialRDD import CircleRDD, PolygonRDD
+from sedona.spark import Adapter
class TestAdapter(TestBase):
diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py
index 22ee40c799..6676063f7d 100644
--- a/python/tests/sql/test_dataframe_api.py
+++ b/python/tests/sql/test_dataframe_api.py
@@ -16,7 +16,6 @@
# under the License.
from math import radians
import os
-import threading
import concurrent.futures
from typing import Callable, Tuple
@@ -26,15 +25,12 @@ from pyspark.sql import functions as f
from shapely.geometry.base import BaseGeometry
from tests.test_base import TestBase
-from sedona.core.geom.geography import Geography
-from sedona.sql import st_aggregates as sta
-from sedona.sql import st_constructors as stc
-from sedona.sql import st_functions as stf
-from sedona.sql import st_predicates as stp
-from sedona.sql.st_aggregates import *
-from sedona.sql.st_constructors import *
-from sedona.sql.st_functions import *
-from sedona.sql.st_predicates import *
+from sedona.spark.core.geom.geography import Geography
+from sedona.spark.sql import st_aggregates as sta
+from sedona.spark.sql import st_constructors as stc
+from sedona.spark.sql import st_functions as stf
+from sedona.spark.sql import st_predicates as stp
+from sedona.spark.sql.st_functions import *
test_configurations = [
# constructors
diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py
index 93ab59f24f..0d6ac29330 100644
--- a/python/tests/sql/test_function.py
+++ b/python/tests/sql/test_function.py
@@ -34,7 +34,7 @@ from tests.sql.resource.sample_data import (
)
from tests.test_base import TestBase
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
class TestPredicateJoin(TestBase):
diff --git a/python/tests/sql/test_geography.py b/python/tests/sql/test_geography.py
index 3105b6a8b4..5721248342 100644
--- a/python/tests/sql/test_geography.py
+++ b/python/tests/sql/test_geography.py
@@ -15,12 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-import pytest
from pyspark.sql.functions import expr
from pyspark.sql.types import StructType
from shapely.wkt import loads as wkt_loads
-from sedona.core.geom.geography import Geography
-from sedona.sql.types import GeographyType
+from sedona.spark.core.geom.geography import Geography
+from sedona.spark.sql.types import GeographyType
from tests.test_base import TestBase
diff --git a/python/tests/sql/test_shapefile.py b/python/tests/sql/test_shapefile.py
index a6d5a578c6..46629360f4 100644
--- a/python/tests/sql/test_shapefile.py
+++ b/python/tests/sql/test_shapefile.py
@@ -18,9 +18,8 @@
import datetime
import os.path
-import pytest
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
class TestShapefile(TestBase):
diff --git a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
index 66bbaa71c8..cb1acb46e6 100644
--- a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
+++ b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
@@ -19,12 +19,13 @@ import os
from pyspark.sql.types import IntegerType, StringType, StructField, StructType
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter
-from sedona.core.SpatialRDD import PointRDD
-from sedona.sql.types import GeometryType
+from sedona.spark.core.enums import FileDataSplitter
+from sedona.spark.core.SpatialRDD import PointRDD
+from sedona.spark.sql.types import GeometryType
point_input_path = os.path.join(tests_resource, "arealm-small.csv")
diff --git a/python/tests/sql/test_st_function_imports.py b/python/tests/sql/test_st_function_imports.py
index f0c38f6b07..c09d1b8ae5 100644
--- a/python/tests/sql/test_st_function_imports.py
+++ b/python/tests/sql/test_st_function_imports.py
@@ -22,7 +22,12 @@ from tests.test_base import TestBase
class TestStFunctionImport(TestBase):
def test_import(self):
- from sedona.sql import ST_Contains, ST_Distance, ST_Envelope_Aggr, ST_Point
+ from sedona.spark.sql import (
+ ST_Contains,
+ ST_Distance,
+ ST_Envelope_Aggr,
+ ST_Point,
+ )
ST_Distance
ST_Point
diff --git a/python/tests/sql/test_structured_adapter.py b/python/tests/sql/test_structured_adapter.py
index f14a48f33e..757bc46550 100644
--- a/python/tests/sql/test_structured_adapter.py
+++ b/python/tests/sql/test_structured_adapter.py
@@ -19,10 +19,10 @@ import tempfile
from pyspark.sql import DataFrame
-from sedona.core.SpatialRDD import CircleRDD
-from sedona.core.enums import GridType
-from sedona.core.spatialOperator import JoinQueryRaw
-from sedona.utils.structured_adapter import StructuredAdapter
+from sedona.spark.core.SpatialRDD import CircleRDD
+from sedona.spark.core.enums import GridType
+from sedona.spark.core.spatialOperator import JoinQueryRaw
+from sedona.spark.utils.structured_adapter import StructuredAdapter
from tests.test_base import TestBase
diff --git a/python/tests/stac/test_client.py b/python/tests/stac/test_client.py
index b8b2beed8a..636cfcef8e 100644
--- a/python/tests/stac/test_client.py
+++ b/python/tests/stac/test_client.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.stac.client import Client
+from sedona.spark.stac.client import Client
from pyspark.sql import DataFrame
from tests.test_base import TestBase
@@ -124,8 +124,6 @@ class TestStacClient(TestBase):
assert len(list(items)) == 0
def test_search_with_YYYY(self) -> None:
- from datetime import datetime
-
client = Client.open(STAC_URLS["PLANETARY-COMPUTER"])
items = client.search(
collection_id="aster-l1t",
diff --git a/python/tests/stac/test_collection_client.py b/python/tests/stac/test_collection_client.py
index aa9a15f4eb..bc08cabcd5 100644
--- a/python/tests/stac/test_collection_client.py
+++ b/python/tests/stac/test_collection_client.py
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.stac.client import Client
-from sedona.stac.collection_client import CollectionClient
+from sedona.spark.stac.client import Client
+from sedona.spark.stac.collection_client import CollectionClient
from tests.test_base import TestBase
diff --git a/python/tests/stats/test_dbscan.py b/python/tests/stats/test_dbscan.py
index 29080d93fb..1b74044755 100644
--- a/python/tests/stats/test_dbscan.py
+++ b/python/tests/stats/test_dbscan.py
@@ -20,9 +20,9 @@ import pytest
from sklearn.cluster import DBSCAN as sklearnDBSCAN
from tests.test_base import TestBase
-from sedona.sql.st_constructors import ST_MakePoint
-from sedona.sql.st_functions import ST_Buffer
-from sedona.stats.clustering.dbscan import dbscan
+from sedona.spark.sql.st_constructors import ST_MakePoint
+from sedona.spark.sql.st_functions import ST_Buffer
+from sedona.spark.stats import dbscan
class TestDBScan(TestBase):
diff --git a/python/tests/stats/test_getis_ord.py b/python/tests/stats/test_getis_ord.py
index d40ef4056a..27cbc25de8 100644
--- a/python/tests/stats/test_getis_ord.py
+++ b/python/tests/stats/test_getis_ord.py
@@ -18,11 +18,12 @@
from esda.getisord import G_Local
from libpysal.weights import DistanceBand
from pyspark.sql import functions as f
+
+from sedona.spark.stats import g_local
+from sedona.spark.stats import add_distance_band_column
from tests.test_base import TestBase
-from sedona.sql.st_constructors import ST_MakePoint
-from sedona.stats.hotspot_detection.getis_ord import g_local
-from sedona.stats.weighting import add_distance_band_column
+from sedona.spark.sql.st_constructors import ST_MakePoint
class TestGetisOrd(TestBase):
diff --git a/python/tests/stats/test_local_outlier_factor.py b/python/tests/stats/test_local_outlier_factor.py
index 52ec860a05..ed03af90c4 100644
--- a/python/tests/stats/test_local_outlier_factor.py
+++ b/python/tests/stats/test_local_outlier_factor.py
@@ -23,9 +23,9 @@ from pyspark.sql.types import DoubleType, IntegerType, StructField, StructType
from sklearn.neighbors import LocalOutlierFactor
from tests.test_base import TestBase
-from sedona.sql.st_constructors import ST_MakePoint
-from sedona.sql.st_functions import ST_X, ST_Y
-from sedona.stats.outlier_detection.local_outlier_factor import local_outlier_factor
+from sedona.spark.sql.st_constructors import ST_MakePoint
+from sedona.spark.sql.st_functions import ST_X, ST_Y
+from sedona.spark.stats import local_outlier_factor
class TestLOF(TestBase):
diff --git a/python/tests/stats/test_weighting.py b/python/tests/stats/test_weighting.py
index 5dadcaef46..82b30a208d 100644
--- a/python/tests/stats/test_weighting.py
+++ b/python/tests/stats/test_weighting.py
@@ -18,8 +18,8 @@
import pyspark.sql.functions as f
from tests.test_base import TestBase
-from sedona.sql.st_constructors import ST_MakePoint
-from sedona.stats.weighting import (
+from sedona.spark.sql.st_constructors import ST_MakePoint
+from sedona.spark.stats.weighting import (
add_binary_distance_band_column,
add_distance_band_column,
)
diff --git a/python/tests/streaming/spark/test_constructor_functions.py b/python/tests/streaming/spark/test_constructor_functions.py
index 354a5ff2a1..2cfe2cf1c4 100644
--- a/python/tests/streaming/spark/test_constructor_functions.py
+++ b/python/tests/streaming/spark/test_constructor_functions.py
@@ -27,7 +27,7 @@ from tests import tests_resource
from tests.streaming.spark.cases_builder import SuiteContainer
from tests.test_base import TestBase
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
SCHEMA = StructType([StructField("geom", GeometryType())])
diff --git a/python/tests/test_assign_raw_spatial_rdd.py b/python/tests/test_assign_raw_spatial_rdd.py
index 303ded9ace..1a8b110498 100644
--- a/python/tests/test_assign_raw_spatial_rdd.py
+++ b/python/tests/test_assign_raw_spatial_rdd.py
@@ -23,7 +23,7 @@ from tests.properties.point_properties import (
)
from tests.test_base import TestBase
-from sedona.core.SpatialRDD import CircleRDD, PointRDD
+from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD
class TestSpatialRddAssignment(TestBase):
diff --git a/python/tests/test_base.py b/python/tests/test_base.py
index 2582b9d4bf..d2794ac8aa 100644
--- a/python/tests/test_base.py
+++ b/python/tests/test_base.py
@@ -23,7 +23,7 @@ import pyspark
from pyspark.sql import DataFrame
from sedona.spark import *
-from sedona.utils.decorators import classproperty
+from sedona.spark.utils.decorators import classproperty
SPARK_REMOTE = os.getenv("SPARK_REMOTE")
EXTRA_JARS = os.getenv("SEDONA_PYTHON_EXTRA_JARS")
diff --git a/python/tests/test_circle.py b/python/tests/test_circle.py
index 688a399630..508de9728a 100644
--- a/python/tests/test_circle.py
+++ b/python/tests/test_circle.py
@@ -20,8 +20,8 @@ import shapely
from shapely import wkt
from shapely.geometry import Point
-from sedona.core.geom.circle import Circle
-from sedona.core.geom.envelope import Envelope
+from sedona.spark.core.geom.circle import Circle
+from sedona.spark.core.geom.envelope import Envelope
class TestCircle:
diff --git a/python/tests/test_multiple_meta.py b/python/tests/test_multiple_meta.py
index c37cc72e82..accaa5070b 100644
--- a/python/tests/test_multiple_meta.py
+++ b/python/tests/test_multiple_meta.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from sedona.utils.meta import MultipleMeta
+from sedona.spark.utils.meta import MultipleMeta
class TestMultipleMeta:
diff --git a/python/tests/test_scala_example.py b/python/tests/test_scala_example.py
index d2e1a33502..76c4831ed3 100644
--- a/python/tests/test_scala_example.py
+++ b/python/tests/test_scala_example.py
@@ -19,13 +19,14 @@ import os
from pyspark import StorageLevel
from shapely.geometry import Point
+
+from tests import tests_resource
from tests.test_base import TestBase
-from tests.tools import tests_resource
-from sedona.core.enums import FileDataSplitter, GridType, IndexType
-from sedona.core.geom.envelope import Envelope
-from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery
-from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD
+from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType
+from sedona.spark.core.geom.envelope import Envelope
+from sedona.spark.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery
+from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD
point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv")
@@ -77,207 +78,208 @@ class TestScalaExample(TestBase):
object_rdd, range_query_window, False, False
).count()
- def test_spatial_range_query_using_index(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- object_rdd.buildIndex(point_rdd_index_type, False)
- object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY)
- assert object_rdd.indexedRawRDD.is_cached
-
- for _ in range(each_query_loop_times):
- result_size = RangeQuery.SpatialRangeQuery(
- object_rdd, range_query_window, False, True
- ).count
-
- def test_spatial_knn_query(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- object_rdd.rawJvmSpatialRDD.persist(StorageLevel.MEMORY_ONLY)
-
- for _ in range(each_query_loop_times):
- result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, False)
-
- def test_spatial_knn_query_using_index(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- object_rdd.buildIndex(point_rdd_index_type, False)
- object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY)
-
- for _ in range(each_query_loop_times):
- result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, True)
-
- def test_spatial_join_query(self):
- query_window_rdd = PolygonRDD(
- self.sc,
- polygon_rdd_input_location,
- polygon_rdd_start_offset,
- polygon_rdd_end_offset,
- polygon_rdd_splitter,
- True,
- )
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
-
- object_rdd.spatialPartitioning(join_query_partitioning_type)
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
- query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
-
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.SpatialJoinQuery(
- object_rdd, query_window_rdd, False, True
- ).count()
-
- def test_spatial_join_using_index(self):
- query_window_rdd = PolygonRDD(
- self.sc,
- polygon_rdd_input_location,
- polygon_rdd_start_offset,
- polygon_rdd_end_offset,
- polygon_rdd_splitter,
- True,
- )
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
-
- object_rdd.spatialPartitioning(join_query_partitioning_type)
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.buildIndex(point_rdd_index_type, True)
-
- object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
- query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
-
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.SpatialJoinQuery(
- object_rdd, query_window_rdd, True, False
- ).count()
-
- def test_distance_join_query(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- query_window_rdd = CircleRDD(object_rdd, 0.1)
-
- object_rdd.spatialPartitioning(GridType.QUADTREE)
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
- assert object_rdd.spatialPartitionedRDD.is_cached
-
- query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
-
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.DistanceJoinQuery(
- object_rdd, query_window_rdd, False, True
- ).count()
-
- def test_distance_join_using_index(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
-
- query_window_rdd = CircleRDD(object_rdd, 0.1)
-
- object_rdd.spatialPartitioning(GridType.QUADTREE)
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.buildIndex(IndexType.RTREE, True)
-
- object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
- query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
- assert object_rdd.indexedRDD.is_cached
- assert query_window_rdd.spatialPartitionedRDD.is_cached
-
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.DistanceJoinQuery(
- object_rdd, query_window_rdd, True, True
- ).count()
-
- def test_indexed_rdd_assignment(self):
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- query_window_rdd = CircleRDD(object_rdd, 0.1)
- object_rdd.analyze()
- object_rdd.spatialPartitioning(GridType.QUADTREE)
- object_rdd.buildIndex(IndexType.QUADTREE, True)
-
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.buildIndex(IndexType.RTREE, False)
-
- object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
- query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
- query_window_rdd.jvmSpatialPartitionedRDD.count()
- object_rdd.indexedRDD.count()
-
- import time
-
- start = time.time()
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.DistanceJoinQuery(
- object_rdd, query_window_rdd, True, True
- ).count()
- diff = time.time() - start
-
- object_rdd = PointRDD(
- self.sc,
- point_rdd_input_location,
- point_rdd_offset,
- point_rdd_splitter,
- True,
- )
- query_window_rdd = CircleRDD(object_rdd, 0.1)
-
- object_rdd.analyze()
- object_rdd.spatialPartitioning(GridType.QUADTREE)
- object_rdd.buildIndex(IndexType.QUADTREE, True)
-
- query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
-
- object_rdd.buildIndex(IndexType.RTREE, False)
-
- start1 = time.time()
- for _ in range(each_query_loop_times):
- result_size = JoinQuery.DistanceJoinQuery(
- object_rdd, query_window_rdd, True, True
- ).count()
+ #
+ # def test_spatial_range_query_using_index(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # object_rdd.buildIndex(point_rdd_index_type, False)
+ # object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY)
+ # assert object_rdd.indexedRawRDD.is_cached
+ #
+ # for _ in range(each_query_loop_times):
+ # result_size = RangeQuery.SpatialRangeQuery(
+ # object_rdd, range_query_window, False, True
+ # ).count
+ #
+ # def test_spatial_knn_query(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # object_rdd.rawJvmSpatialRDD.persist(StorageLevel.MEMORY_ONLY)
+ #
+ # for _ in range(each_query_loop_times):
+ # result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, False)
+ #
+ # def test_spatial_knn_query_using_index(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # object_rdd.buildIndex(point_rdd_index_type, False)
+ # object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY)
+ #
+ # for _ in range(each_query_loop_times):
+ # result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, True)
+ #
+ # def test_spatial_join_query(self):
+ # query_window_rdd = PolygonRDD(
+ # self.sc,
+ # polygon_rdd_input_location,
+ # polygon_rdd_start_offset,
+ # polygon_rdd_end_offset,
+ # polygon_rdd_splitter,
+ # True,
+ # )
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ #
+ # object_rdd.spatialPartitioning(join_query_partitioning_type)
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ #
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.SpatialJoinQuery(
+ # object_rdd, query_window_rdd, False, True
+ # ).count()
+ #
+ # def test_spatial_join_using_index(self):
+ # query_window_rdd = PolygonRDD(
+ # self.sc,
+ # polygon_rdd_input_location,
+ # polygon_rdd_start_offset,
+ # polygon_rdd_end_offset,
+ # polygon_rdd_splitter,
+ # True,
+ # )
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ #
+ # object_rdd.spatialPartitioning(join_query_partitioning_type)
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.buildIndex(point_rdd_index_type, True)
+ #
+ # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ #
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.SpatialJoinQuery(
+ # object_rdd, query_window_rdd, True, False
+ # ).count()
+ #
+ # def test_distance_join_query(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # query_window_rdd = CircleRDD(object_rdd, 0.1)
+ #
+ # object_rdd.spatialPartitioning(GridType.QUADTREE)
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # assert object_rdd.spatialPartitionedRDD.is_cached
+ #
+ # query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ #
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.DistanceJoinQuery(
+ # object_rdd, query_window_rdd, False, True
+ # ).count()
+ #
+ # def test_distance_join_using_index(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ #
+ # query_window_rdd = CircleRDD(object_rdd, 0.1)
+ #
+ # object_rdd.spatialPartitioning(GridType.QUADTREE)
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.buildIndex(IndexType.RTREE, True)
+ #
+ # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # assert object_rdd.indexedRDD.is_cached
+ # assert query_window_rdd.spatialPartitionedRDD.is_cached
+ #
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.DistanceJoinQuery(
+ # object_rdd, query_window_rdd, True, True
+ # ).count()
+ #
+ # def test_indexed_rdd_assignment(self):
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # query_window_rdd = CircleRDD(object_rdd, 0.1)
+ # object_rdd.analyze()
+ # object_rdd.spatialPartitioning(GridType.QUADTREE)
+ # object_rdd.buildIndex(IndexType.QUADTREE, True)
+ #
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.buildIndex(IndexType.RTREE, False)
+ #
+ # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY)
+ # query_window_rdd.jvmSpatialPartitionedRDD.count()
+ # object_rdd.indexedRDD.count()
+ #
+ # import time
+ #
+ # start = time.time()
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.DistanceJoinQuery(
+ # object_rdd, query_window_rdd, True, True
+ # ).count()
+ # diff = time.time() - start
+ #
+ # object_rdd = PointRDD(
+ # self.sc,
+ # point_rdd_input_location,
+ # point_rdd_offset,
+ # point_rdd_splitter,
+ # True,
+ # )
+ # query_window_rdd = CircleRDD(object_rdd, 0.1)
+ #
+ # object_rdd.analyze()
+ # object_rdd.spatialPartitioning(GridType.QUADTREE)
+ # object_rdd.buildIndex(IndexType.QUADTREE, True)
+ #
+ # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
+ #
+ # object_rdd.buildIndex(IndexType.RTREE, False)
+ #
+ # start1 = time.time()
+ # for _ in range(each_query_loop_times):
+ # result_size = JoinQuery.DistanceJoinQuery(
+ # object_rdd, query_window_rdd, True, True
+ # ).count()
diff --git a/python/tests/tools.py b/python/tests/tools.py
index 0e9be54aec..bd454d3824 100644
--- a/python/tests/tools.py
+++ b/python/tests/tools.py
@@ -14,17 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from os import path
-
from shapely.geometry import Point
-from sedona.utils.spatial_rdd_parser import GeoData
-
-tests_path = path.abspath(path.join(__file__, "../../../spark/common/src/test/"))
-tests_resource = path.abspath(
- path.join(__file__, "../../../spark/common/src/test/resources/")
-)
+from sedona.spark.utils.spatial_rdd_parser import GeoData
def distance_sorting_functions(geo_data: GeoData, query_point: Point):
diff --git a/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py b/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py
index 91314c7406..8afa92fffe 100644
--- a/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py
+++ b/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py
@@ -17,8 +17,8 @@
import pytest
-from sedona.sql.types import GeometryType
-from sedona.utils.geoarrow import create_spatial_dataframe
+from sedona.spark.sql.types import GeometryType
+from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe
from tests.test_base import TestBase
import geopandas as gpd
import pyspark
diff --git a/python/tests/utils/test_crs_transformation.py b/python/tests/utils/test_crs_transformation.py
index b751857fbf..b4a57cf602 100644
--- a/python/tests/utils/test_crs_transformation.py
+++ b/python/tests/utils/test_crs_transformation.py
@@ -15,15 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-from tests.properties.crs_transform import *
-from tests.properties.polygon_properties import grid_type
+from tests.properties.crs_transform import (
+ input_location,
+ offset,
+ splitter,
+ loop_times,
+ query_envelope,
+)
from tests.test_base import TestBase
-from tests.tools import distance_sorting_functions
-from sedona.core.enums import GridType
-from sedona.core.geom.circle import Circle
-from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery
-from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD
+from sedona.spark.core.spatialOperator import RangeQuery
+from sedona.spark.core.SpatialRDD import PointRDD
class TestCrsTransformation(TestBase):
diff --git a/python/tests/utils/test_geo_spark_meta.py b/python/tests/utils/test_geo_spark_meta.py
index 14f6464a8d..4ee1b34ddd 100644
--- a/python/tests/utils/test_geo_spark_meta.py
+++ b/python/tests/utils/test_geo_spark_meta.py
@@ -17,7 +17,7 @@
from tests.test_base import TestBase
-from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version
+from sedona.spark.core.jvm.config import SedonaMeta, is_greater_or_equal_version
class TestGeoSparkMeta(TestBase):
diff --git a/python/tests/utils/test_geoarrow.py b/python/tests/utils/test_geoarrow.py
index 28499c401c..6ed3b08486 100644
--- a/python/tests/utils/test_geoarrow.py
+++ b/python/tests/utils/test_geoarrow.py
@@ -22,7 +22,7 @@ import pyproj
import pytest
from tests.test_base import TestBase
from pyspark.sql.types import StringType, StructType
-from sedona.utils.geoarrow import (
+from sedona.spark.geoarrow.geoarrow import (
dataframe_to_arrow,
unique_srid_from_ewkb,
wrap_geoarrow_field,
diff --git a/python/tests/utils/test_geometry_serde.py b/python/tests/utils/test_geometry_serde.py
index 661aa169dd..7be419ea64 100644
--- a/python/tests/utils/test_geometry_serde.py
+++ b/python/tests/utils/test_geometry_serde.py
@@ -29,7 +29,7 @@ from shapely.geometry import (
from shapely.wkt import loads as wkt_loads
from tests.test_base import TestBase
-from sedona.sql.types import GeometryType
+from sedona.spark.sql.types import GeometryType
class TestGeometrySerde(TestBase):
diff --git a/python/tests/utils/test_geomserde_speedup.py b/python/tests/utils/test_geomserde_speedup.py
index ea2edc4a1e..f4134ff76c 100644
--- a/python/tests/utils/test_geomserde_speedup.py
+++ b/python/tests/utils/test_geomserde_speedup.py
@@ -27,7 +27,7 @@ from shapely.geometry import (
from shapely.geometry.base import BaseGeometry
from shapely.wkt import loads as wkt_loads
-from sedona.utils import geometry_serde
+from sedona.spark.utils import geometry_serde
class TestGeomSerdeSpeedup:
diff --git a/python/tests/utils/test_pandas_arrow_udf.py b/python/tests/utils/test_pandas_arrow_udf.py
index c0d723d214..6db0b25268 100644
--- a/python/tests/utils/test_pandas_arrow_udf.py
+++ b/python/tests/utils/test_pandas_arrow_udf.py
@@ -16,8 +16,8 @@
# under the License.
-from sedona.sql.types import GeometryType
-from sedona.sql.functions import sedona_vectorized_udf, SedonaUDFType
+from sedona.spark.sql.types import GeometryType
+from sedona.spark.sql.functions import sedona_vectorized_udf, SedonaUDFType
from tests import chicago_crimes_input_location
from tests.test_base import TestBase
import pyspark.sql.functions as f
diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala
index 18ad4dfebd..f9eea7f98c 100644
--- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala
@@ -28,7 +28,7 @@ import org.apache.sedona.common.geometryObjects.Geography;
class GeographyUDT extends UserDefinedType[Geography] {
override def sqlType: DataType = BinaryType
- override def pyUDT: String = "sedona.sql.types.GeographyType"
+ override def pyUDT: String = "sedona.spark.sql.types.GeographyType"
override def userClass: Class[Geography] = classOf[Geography]
diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
index 65f347307c..20ed85d52e 100644
--- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
@@ -28,7 +28,7 @@ import org.locationtech.jts.geom.Geometry
class GeometryUDT extends UserDefinedType[Geometry] {
override def sqlType: DataType = BinaryType
- override def pyUDT: String = "sedona.sql.types.GeometryType"
+ override def pyUDT: String = "sedona.spark.sql.types.GeometryType"
override def userClass: Class[Geometry] = classOf[Geometry]
diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
index 8ebc2762a6..ee05cd1521 100644
--- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
@@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GridCoverage2D
class RasterUDT extends UserDefinedType[GridCoverage2D] {
override def sqlType: DataType = BinaryType
- override def pyUDT: String = "sedona.sql.types.RasterType"
+ override def pyUDT: String = "sedona.spark.sql.types.RasterType"
// A reasonable size for small in-db rasters. This is used by the optimizer to decide whether to
// broadcast the dataframe or not.
(sedona) 02/02: SEDONA-725 rearrange the spark module
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imbruced pushed a commit to branch SEDONA-725-restructure-spark-python-package
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit db7af60eaec022ad832ca2b99ba623ee6cf40fd3
Author: pawelkocinski <pa...@gmail.com>
AuthorDate: Tue Apr 29 22:09:26 2025 +0200
SEDONA-725 rearrange the spark module
---
.github/workflows/pyflink.yml | 78 ---------------------------
python/sedona/flink/__init__.py | 14 -----
python/sedona/flink/context.py | 35 ------------
python/setup.py | 1 -
python/tests/flink/conftest.py | 43 ---------------
python/tests/flink/test_flink_registration.py | 45 ----------------
6 files changed, 216 deletions(-)
diff --git a/.github/workflows/pyflink.yml b/.github/workflows/pyflink.yml
deleted file mode 100644
index 7e9732190d..0000000000
--- a/.github/workflows/pyflink.yml
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Sedona Pyflink Test
-
-on:
- push:
- branches:
- - master
- paths:
- - 'common/**'
- - 'flink/**'
- - 'flink-shaded/**'
- - 'pom.xml'
- - 'python/**'
- - '.github/workflows/pyflink.yml'
- pull_request:
- branches:
- - '*'
- paths:
- - 'common/**'
- - 'flink/**'
- - 'flink-shaded/**'
- - 'pom.xml'
- - 'python/**'
- - '.github/workflows/pyflink.yml'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- test:
- runs-on: ubuntu-22.04
- strategy:
- matrix:
- include:
- - python: '3.10'
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: '8'
- - uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python }}
- - run: sudo apt-get -y install python3-pip python-dev-is-python3
- - run: mvn package -pl "org.apache.sedona:sedona-flink-shaded_2.12" -am -DskipTests
- - run: sudo pip3 install -U setuptools
- - run: sudo pip3 install -U wheel
- - run: sudo pip3 install -U virtualenvwrapper
- - run: python3 -m pip install uv
- - run: cd python
- - run: rm pyproject.toml
- - run: uv init --no-workspace
- - run: uv add apache-flink==1.20.1 shapely attr setuptools
- - run: uv add pytest --dev
- - run: |
- wget https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/1.7.1-28.5/geotools-wrapper-1.7.1-28.5.jar
- export SEDONA_PYFLINK_EXTRA_JARS=${PWD}/$(find flink-shaded/target -name sedona-flink*.jar),${PWD}/geotools-wrapper-1.7.1-28.5.jar
- pwd
- ls -l
- (cd python; PYTHONPATH=$(pwd) uv run pytest -v -m flink -s ./tests/flink)
diff --git a/python/sedona/flink/__init__.py b/python/sedona/flink/__init__.py
deleted file mode 100644
index 4219449814..0000000000
--- a/python/sedona/flink/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import logging
-import warnings
-
-try:
- import pyflink
- from sedona.flink.context import SedonaContext
-except ImportError:
- warnings.warn(
- "Apache Sedona requires Pyflink. Please install PyFlink before using Sedona flink.",
- DeprecationWarning,
- stacklevel=2,
- )
-
-__all__ = ["SedonaContext"]
diff --git a/python/sedona/flink/context.py b/python/sedona/flink/context.py
deleted file mode 100644
index 2f14d20b19..0000000000
--- a/python/sedona/flink/context.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import warnings
-
-try:
- from pyflink.table import EnvironmentSettings, StreamTableEnvironment
- from pyflink.datastream import StreamExecutionEnvironment
- from pyflink.java_gateway import get_gateway
-except ImportError:
- StreamTableEnvironment = None
- StreamExecutionEnvironment = None
- EnvironmentSettings = None
- warnings.warn(
- "Apache Sedona requires Pyflink. Please install PyFlink before using Sedona flink.",
- DeprecationWarning,
- stacklevel=2,
- )
-
-
-class SedonaContext:
-
- @classmethod
- def create(
- cls, env: StreamExecutionEnvironment, settings: EnvironmentSettings
- ) -> StreamTableEnvironment:
- table_env = StreamTableEnvironment.create(env, settings)
- gateway = get_gateway()
-
- flink_sedona_context = gateway.jvm.org.apache.sedona.flink.SedonaContext
-
- table_env_j = flink_sedona_context.create(
- env._j_stream_execution_environment, table_env._j_tenv
- )
-
- table_env._j_tenv = table_env_j
-
- return table_env
diff --git a/python/setup.py b/python/setup.py
index 695966631e..214a1f0029 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -63,7 +63,6 @@ setup(
"spark": ["pyspark>=2.3.0"],
"pydeck-map": ["geopandas", "pydeck==0.8.0"],
"kepler-map": ["geopandas", "keplergl==0.3.2"],
- "flink": ["apache-flink>=1.19.0"],
"all": [
"pyspark>=2.3.0",
"geopandas",
diff --git a/python/tests/flink/conftest.py b/python/tests/flink/conftest.py
deleted file mode 100644
index 2163f3c514..0000000000
--- a/python/tests/flink/conftest.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-
-import pytest
-
-
-EXTRA_JARS = os.getenv("SEDONA_PYFLINK_EXTRA_JARS")
-
-
-def has_pyflink():
- try:
- import pyflink
- except ImportError:
- return False
- return True
-
-
-if has_pyflink():
- from sedona.flink import SedonaContext
-
- try:
- from pyflink.datastream import StreamExecutionEnvironment
- from pyflink.table import EnvironmentSettings, StreamTableEnvironment
- except ImportError:
- pytest.skip("PyFlink is not installed. Skipping tests that require PyFlink.")
-
- @pytest.fixture(scope="module")
- def flink_settings():
- return EnvironmentSettings.in_streaming_mode()
-
- @pytest.fixture(scope="module")
- def stream_env() -> StreamExecutionEnvironment:
- env = StreamExecutionEnvironment.get_execution_environment()
- jars = EXTRA_JARS.split(",") if EXTRA_JARS else []
- for jar in jars:
- env.add_jars(f"file://{jar}")
-
- return env
-
- @pytest.fixture(scope="module")
- def table_env(
- stream_env: StreamExecutionEnvironment, flink_settings: EnvironmentSettings
- ) -> StreamTableEnvironment:
- return SedonaContext.create(stream_env, flink_settings)
diff --git a/python/tests/flink/test_flink_registration.py b/python/tests/flink/test_flink_registration.py
deleted file mode 100644
index 02c6c08261..0000000000
--- a/python/tests/flink/test_flink_registration.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from shapely.wkb import loads
-import pytest
-
-
-@pytest.mark.flink
-def test_register(table_env):
- result = (
- table_env.sql_query("SELECT ST_ASBinary(ST_Point(1.0, 2.0))")
- .execute()
- .collect()
- )
-
- assert 1 == len([el for el in result])
-
-
-@pytest.mark.flink
-def test_register_udf(table_env):
- from pyflink.table.udf import ScalarFunction, udf
-
- class Buffer(ScalarFunction):
- def eval(self, s):
- geom = loads(s)
- return geom.buffer(1).wkb
-
- table_env.create_temporary_function(
- "ST_BufferPython", udf(Buffer(), result_type="Binary")
- )
-
- buffer_table = table_env.sql_query(
- "SELECT ST_BufferPython(ST_ASBinary(ST_Point(1.0, 2.0))) AS buffer"
- )
-
- table_env.create_temporary_view("buffer_table", buffer_table)
-
- result = (
- table_env.sql_query("SELECT ST_Area(ST_GeomFromWKB(buffer)) FROM buffer_table")
- .execute()
- .collect()
- )
-
- items = [el for el in result]
- area = items[0][0]
-
- assert 3.12 < area < 3.14
- assert 1 == len(items)