You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2024/09/23 22:14:09 UTC
(sedona) branch master updated: [SEDONA-657] pre-commit add Python black hook; format Python with black (#1593)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 0f2d0e744 [SEDONA-657] pre-commit add Python black hook; format Python with black (#1593)
0f2d0e744 is described below
commit 0f2d0e744d79a5c6c2ea57bb7f46ab8e47a80553
Author: John Bampton <jb...@users.noreply.github.com>
AuthorDate: Tue Sep 24 08:14:04 2024 +1000
[SEDONA-657] pre-commit add Python black hook; format Python with black (#1593)
* [SEDONA-657] pre-commit add Python black hook; format Python with black
Standardizes all the Python code to black format
https://black.readthedocs.io/en/stable/integrations/source_version_control.html
https://www.flake8rules.com/
* Add black jupyter
* Remove ruff
---
.github/linters/ruff.toml | 77 -
.pre-commit-config.yaml | 9 +-
docs/usecases/ApacheSedonaCore.ipynb | 88 +-
docs/usecases/ApacheSedonaRaster.ipynb | 118 +-
docs/usecases/ApacheSedonaSQL.ipynb | 120 +-
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 61 +-
docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb | 116 +-
.../usecases/contrib/ApacheSedonaImageFilter.ipynb | 133 +-
docs/usecases/contrib/DownloadImageFromGEE.ipynb | 40 +-
.../contrib/NdviSentinelApacheSedona.ipynb | 265 +--
.../contrib/PostgresqlConnectionApacheSedona.ipynb | 57 +-
.../contrib/VectorAnalisisApacheSedona.ipynb | 291 ++--
docs/usecases/contrib/foot-traffic.ipynb | 555 ++++---
docs/usecases/utilities.py | 231 +--
python/sedona/core/SpatialRDD/__init__.py | 7 +-
python/sedona/core/SpatialRDD/circle_rdd.py | 17 +-
python/sedona/core/SpatialRDD/linestring_rdd.py | 60 +-
python/sedona/core/SpatialRDD/point_rdd.py | 63 +-
python/sedona/core/SpatialRDD/polygon_rdd.py | 58 +-
python/sedona/core/SpatialRDD/rectangle_rdd.py | 60 +-
python/sedona/core/SpatialRDD/spatial_rdd.py | 73 +-
python/sedona/core/enums/file_data_splitter.py | 6 +-
python/sedona/core/enums/grid_type.py | 2 +-
python/sedona/core/enums/index_type.py | 6 +-
python/sedona/core/enums/spatial.py | 2 +-
python/sedona/core/formatMapper/__init__.py | 2 +-
python/sedona/core/formatMapper/disc_utils.py | 6 +-
python/sedona/core/formatMapper/geo_json_reader.py | 30 +-
python/sedona/core/formatMapper/geo_reader.py | 3 +-
.../shapefileParser/shape_file_reader.py | 20 +-
python/sedona/core/formatMapper/wkb_reader.py | 36 +-
python/sedona/core/formatMapper/wkt_reader.py | 33 +-
python/sedona/core/geom/circle.py | 2 +-
python/sedona/core/geom/envelope.py | 2 +-
python/sedona/core/geom/shapely1/circle.py | 46 +-
python/sedona/core/geom/shapely1/envelope.py | 48 +-
python/sedona/core/geom/shapely2/circle.py | 25 +-
python/sedona/core/geom/shapely2/envelope.py | 18 +-
python/sedona/core/jvm/config.py | 7 +-
python/sedona/core/jvm/translate.py | 16 +-
python/sedona/core/spatialOperator/__init__.py | 4 +-
python/sedona/core/spatialOperator/join_params.py | 15 +-
python/sedona/core/spatialOperator/join_query.py | 58 +-
.../sedona/core/spatialOperator/join_query_raw.py | 61 +-
python/sedona/core/spatialOperator/knn_query.py | 20 +-
python/sedona/core/spatialOperator/range_query.py | 13 +-
.../sedona/core/spatialOperator/range_query_raw.py | 21 +-
python/sedona/core/spatialOperator/rdd.py | 15 +-
python/sedona/core/utils.py | 1 +
python/sedona/exceptions.py | 1 +
python/sedona/maps/SedonaMapUtils.py | 16 +-
python/sedona/maps/SedonaPyDeck.py | 200 ++-
python/sedona/raster/awt_raster.py | 13 +-
python/sedona/raster/data_buffer.py | 4 +-
python/sedona/raster/meta.py | 31 +-
python/sedona/raster/raster_serde.py | 93 +-
python/sedona/raster/sample_model.py | 42 +-
python/sedona/raster/sedona_raster.py | 162 +-
python/sedona/raster_utils/SedonaUtils.py | 2 +
python/sedona/register/geo_registrator.py | 2 +
python/sedona/register/java_libs.py | 22 +-
python/sedona/spark/SedonaContext.py | 5 +-
python/sedona/sql/__init__.py | 12 +-
python/sedona/sql/dataframe_api.py | 42 +-
python/sedona/sql/exceptions.py | 1 +
python/sedona/sql/st_aggregates.py | 13 +-
python/sedona/sql/st_constructors.py | 151 +-
python/sedona/sql/st_functions.py | 375 ++++-
python/sedona/sql/st_predicates.py | 38 +-
python/sedona/utils/abstract_parser.py | 4 +-
python/sedona/utils/adapter.py | 55 +-
python/sedona/utils/binary_parser.py | 22 +-
python/sedona/utils/decorators.py | 10 +-
python/sedona/utils/geometry_serde.py | 33 +-
python/sedona/utils/geometry_serde_general.py | 174 +-
python/sedona/utils/jvm.py | 8 +-
python/sedona/utils/meta.py | 23 +-
python/sedona/utils/prep.py | 21 +-
python/sedona/utils/spatial_rdd_parser.py | 40 +-
python/setup.py | 48 +-
python/tests/__init__.py | 40 +-
.../core/test_avoiding_python_jvm_serde_df.py | 140 +-
.../core/test_avoiding_python_jvm_serde_to_rdd.py | 45 +-
python/tests/core/test_core_geom_primitives.py | 4 +-
python/tests/core/test_core_rdd.py | 15 +-
python/tests/core/test_rdd.py | 88 +-
python/tests/core/test_spatial_rdd_from_disc.py | 110 +-
python/tests/format_mapper/test_geo_json_reader.py | 48 +-
.../tests/format_mapper/test_shapefile_reader.py | 50 +-
.../tests/maps/test_sedonakepler_visualization.py | 346 ++--
python/tests/maps/test_sedonapydeck.py | 135 +-
python/tests/properties/linestring_properties.py | 16 +-
python/tests/properties/point_properties.py | 13 +-
python/tests/properties/polygon_properties.py | 8 +-
python/tests/raster/test_meta.py | 8 +-
python/tests/raster/test_pandas_udf.py | 16 +-
python/tests/raster/test_serde.py | 73 +-
python/tests/raster_viz_utils/test_sedonautils.py | 16 +-
python/tests/serialization/test_deserializers.py | 85 +-
.../serialization/test_direct_serialization.py | 27 +-
.../serialization/test_geospark_serializers.py | 7 +-
.../tests/serialization/test_rdd_serialization.py | 26 +-
python/tests/serialization/test_serializers.py | 114 +-
.../tests/serialization/test_with_sc_parellize.py | 18 +-
.../test_join_query_correctness.py | 231 ++-
.../tests/spatial_operator/test_linestring_join.py | 45 +-
.../tests/spatial_operator/test_linestring_knn.py | 8 +-
.../spatial_operator/test_linestring_range.py | 32 +-
python/tests/spatial_operator/test_point_join.py | 90 +-
python/tests/spatial_operator/test_point_knn.py | 32 +-
python/tests/spatial_operator/test_point_range.py | 17 +-
python/tests/spatial_operator/test_polygon_join.py | 35 +-
python/tests/spatial_operator/test_polygon_knn.py | 32 +-
.../tests/spatial_operator/test_polygon_range.py | 34 +-
.../tests/spatial_operator/test_rectangle_join.py | 45 +-
.../tests/spatial_operator/test_rectangle_knn.py | 48 +-
.../tests/spatial_operator/test_rectangle_range.py | 25 +-
python/tests/spatial_rdd/test_circle_rdd.py | 19 +-
python/tests/spatial_rdd/test_linestring_rdd.py | 34 +-
python/tests/spatial_rdd/test_point_rdd.py | 56 +-
python/tests/spatial_rdd/test_polygon_rdd.py | 81 +-
python/tests/spatial_rdd/test_rectangle_rdd.py | 17 +-
python/tests/spatial_rdd/test_spatial_rdd.py | 21 +-
.../tests/spatial_rdd/test_spatial_rdd_writer.py | 18 +-
python/tests/sql/resource/sample_data.py | 28 +-
python/tests/sql/test_adapter.py | 257 ++-
python/tests/sql/test_aggregate_functions.py | 38 +-
python/tests/sql/test_constructor_test.py | 271 ++-
python/tests/sql/test_dataframe_api.py | 1377 ++++++++++++---
python/tests/sql/test_function.py | 1754 ++++++++++++++------
python/tests/sql/test_geoparquet.py | 64 +-
python/tests/sql/test_predicate.py | 221 ++-
python/tests/sql/test_predicate_join.py | 479 ++++--
python/tests/sql/test_shapefile.py | 55 +-
.../sql/test_spatial_rdd_to_spatial_dataframe.py | 13 +-
python/tests/sql/test_st_function_imports.py | 1 +
python/tests/streaming/spark/cases_builder.py | 22 +-
.../streaming/spark/test_constructor_functions.py | 774 +++++----
python/tests/test_assign_raw_spatial_rdd.py | 39 +-
python/tests/test_base.py | 4 +-
python/tests/test_circle.py | 102 +-
python/tests/test_multiple_meta.py | 4 +-
python/tests/test_scala_example.py | 120 +-
python/tests/tools.py | 7 +-
python/tests/utils/test_crs_transformation.py | 20 +-
python/tests/utils/test_geometry_serde.py | 244 ++-
python/tests/utils/test_geomserde_speedup.py | 68 +-
spark-version-converter.py | 87 +-
148 files changed, 8855 insertions(+), 4242 deletions(-)
diff --git a/.github/linters/ruff.toml b/.github/linters/ruff.toml
deleted file mode 100644
index 6302db1e0..000000000
--- a/.github/linters/ruff.toml
+++ /dev/null
@@ -1,77 +0,0 @@
-# Exclude a variety of commonly ignored directories.
-exclude = [
- ".bzr",
- ".direnv",
- ".eggs",
- ".git",
- ".git-rewrite",
- ".hg",
- ".ipynb_checkpoints",
- ".mypy_cache",
- ".nox",
- ".pants.d",
- ".pyenv",
- ".pytest_cache",
- ".pytype",
- ".ruff_cache",
- ".svn",
- ".tox",
- ".venv",
- ".vscode",
- "__pypackages__",
- "_build",
- "buck-out",
- "build",
- "dist",
- "node_modules",
- "site-packages",
- "venv",
-]
-
-# Same as Black.
-line-length = 88
-indent-width = 4
-
-# Assume Python 3.8
-target-version = "py38"
-
-[lint]
-# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
-# McCabe complexity (`C901`) by default.
-select = ["E4", "E7", "E9", "F"]
-ignore = ["E721", "E722", "E731", "F401", "F402", "F403", "F405", "F811", "F821", "F822", "F841", "F901"]
-
-# Allow fix for all enabled rules (when `--fix`) is provided.
-fixable = ["ALL"]
-unfixable = []
-
-# Allow unused variables when underscore-prefixed.
-dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-
-[format]
-# Like Black, use double quotes for strings.
-quote-style = "double"
-
-# Like Black, indent with spaces, rather than tabs.
-indent-style = "space"
-
-# Like Black, respect magic trailing commas.
-skip-magic-trailing-comma = false
-
-# Like Black, automatically detect the appropriate line ending.
-line-ending = "auto"
-
-# Enable auto-formatting of code examples in docstrings. Markdown,
-# reStructuredText code/literal blocks and doctests are all supported.
-#
-# This is currently disabled by default, but it is planned for this
-# to be opt-out in the future.
-docstring-code-format = false
-
-# Set the line length limit used when formatting code snippets in
-# docstrings.
-#
-# This only has an effect when the `docstring-code-format` setting is
-# enabled.
-docstring-code-line-length = "dynamic"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1f3c8cac0..972d26e9c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,6 +10,10 @@ repos:
hooks:
- id: identity
- id: check-hooks-apply
+ - repo: https://github.com/psf/black-pre-commit-mirror
+ rev: 24.8.0
+ hooks:
+ - id: black-jupyter
- repo: https://github.com/codespell-project/codespell
rev: v2.3.0
hooks:
@@ -18,11 +22,6 @@ repos:
description: Check spelling with codespell
args: [--ignore-words=.github/linters/codespell.txt]
exclude: ^docs/image|^spark/common/src/test/resources|^docs/usecases|^tools/maven/scalafmt
- - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.6.5
- hooks:
- - id: ruff
- args: [--config=.github/linters/ruff.toml, --fix]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
hooks:
diff --git a/docs/usecases/ApacheSedonaCore.ipynb b/docs/usecases/ApacheSedonaCore.ipynb
index 69e43e593..e8123163d 100644
--- a/docs/usecases/ApacheSedonaCore.ipynb
+++ b/docs/usecases/ApacheSedonaCore.ipynb
@@ -40,7 +40,7 @@
"from shapely.geometry import Polygon\n",
"\n",
"from sedona.spark import *\n",
- "from sedona.core.geom.envelope import Envelope\n"
+ "from sedona.core.geom.envelope import Envelope"
]
},
{
@@ -182,12 +182,16 @@
}
],
"source": [
- "config = SedonaContext.builder() .\\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,'\n",
- " 'org.datasyslab:geotools-wrapper:1.6.0-28.2,'\n",
- " 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \\\n",
- " getOrCreate()\n",
+ "config = (\n",
+ " SedonaContext.builder()\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.6.0-28.2,\"\n",
+ " \"uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"sedona = SedonaContext.create(config)"
]
@@ -560,7 +564,9 @@
"metadata": {},
"outputs": [],
"source": [
- "point_rdd_to_geo = point_rdd.rawSpatialRDD.map(lambda x: [x.geom, *x.getUserData().split(\"\\t\")])"
+ "point_rdd_to_geo = point_rdd.rawSpatialRDD.map(\n",
+ " lambda x: [x.geom, *x.getUserData().split(\"\\t\")]\n",
+ ")"
]
},
{
@@ -578,7 +584,9 @@
],
"source": [
"point_gdf = gpd.GeoDataFrame(\n",
- " point_rdd_to_geo.collect(), columns=[\"geom\", \"attr1\", \"attr2\", \"attr3\"], geometry=\"geom\"\n",
+ " point_rdd_to_geo.collect(),\n",
+ " columns=[\"geom\", \"attr1\", \"attr2\", \"attr3\"],\n",
+ " geometry=\"geom\",\n",
")"
]
},
@@ -696,9 +704,9 @@
"metadata": {},
"outputs": [],
"source": [
- "spatial_df = Adapter.\\\n",
- " toDf(point_rdd, [\"attr1\", \"attr2\", \"attr3\"], sedona).\\\n",
- " createOrReplaceTempView(\"spatial_df\")\n",
+ "spatial_df = Adapter.toDf(\n",
+ " point_rdd, [\"attr1\", \"attr2\", \"attr3\"], sedona\n",
+ ").createOrReplaceTempView(\"spatial_df\")\n",
"\n",
"spatial_gdf = sedona.sql(\"Select attr1, attr2, attr3, geometry as geom from spatial_df\")"
]
@@ -995,10 +1003,16 @@
"metadata": {},
"outputs": [],
"source": [
- "rectangle_rdd = RectangleRDD(sc, \"data/zcta510-small.csv\", FileDataSplitter.CSV, True, 11)\n",
+ "rectangle_rdd = RectangleRDD(\n",
+ " sc, \"data/zcta510-small.csv\", FileDataSplitter.CSV, True, 11\n",
+ ")\n",
"point_rdd = PointRDD(sc, \"data/arealm-small.csv\", 1, FileDataSplitter.CSV, False, 11)\n",
- "polygon_rdd = PolygonRDD(sc, \"data/primaryroads-polygon.csv\", FileDataSplitter.CSV, True, 11)\n",
- "linestring_rdd = LineStringRDD(sc, \"data/primaryroads-linestring.csv\", FileDataSplitter.CSV, True)"
+ "polygon_rdd = PolygonRDD(\n",
+ " sc, \"data/primaryroads-polygon.csv\", FileDataSplitter.CSV, True, 11\n",
+ ")\n",
+ "linestring_rdd = LineStringRDD(\n",
+ " sc, \"data/primaryroads-linestring.csv\", FileDataSplitter.CSV, True\n",
+ ")"
]
},
{
@@ -1298,7 +1312,7 @@
"schema = StructType(\n",
" [\n",
" StructField(\"geom_left\", GeometryType(), False),\n",
- " StructField(\"geom_right\", GeometryType(), False)\n",
+ " StructField(\"geom_right\", GeometryType(), False),\n",
" ]\n",
")"
]
@@ -1497,7 +1511,9 @@
"metadata": {},
"outputs": [],
"source": [
- "spatial_join_result_non_flat = JoinQuery.SpatialJoinQuery(point_rdd, rectangle_rdd, False, True)"
+ "spatial_join_result_non_flat = JoinQuery.SpatialJoinQuery(\n",
+ " point_rdd, rectangle_rdd, False, True\n",
+ ")"
]
},
{
@@ -1507,7 +1523,9 @@
"outputs": [],
"source": [
"# number of point for each polygon\n",
- "number_of_points = spatial_join_result_non_flat.map(lambda x: [x[0].geom, x[1].__len__()])"
+ "number_of_points = spatial_join_result_non_flat.map(\n",
+ " lambda x: [x[0].geom, x[1].__len__()]\n",
+ ")"
]
},
{
@@ -1516,10 +1534,12 @@
"metadata": {},
"outputs": [],
"source": [
- "schema = StructType([\n",
- " StructField(\"geometry\", GeometryType(), False),\n",
- " StructField(\"number_of_points\", LongType(), False)\n",
- "])"
+ "schema = StructType(\n",
+ " [\n",
+ " StructField(\"geometry\", GeometryType(), False),\n",
+ " StructField(\"number_of_points\", LongType(), False),\n",
+ " ]\n",
+ ")"
]
},
{
@@ -1650,10 +1670,14 @@
],
"source": [
"polygon = Polygon(\n",
- " [(-84.237756, 33.904859), (-84.237756, 34.090426),\n",
- " (-83.833011, 34.090426), (-83.833011, 33.904859),\n",
- " (-84.237756, 33.904859)\n",
- " ])\n",
+ " [\n",
+ " (-84.237756, 33.904859),\n",
+ " (-84.237756, 34.090426),\n",
+ " (-83.833011, 34.090426),\n",
+ " (-83.833011, 33.904859),\n",
+ " (-84.237756, 33.904859),\n",
+ " ]\n",
+ ")\n",
"polygons_nearby = KNNQuery.SpatialKnnQuery(polygon_rdd, polygon, 5, False)"
]
},
@@ -1737,7 +1761,9 @@
"source": [
"query_envelope = Envelope(-85.01, -60.01, 34.01, 50.01)\n",
"\n",
- "result_range_query = RangeQuery.SpatialRangeQuery(linestring_rdd, query_envelope, False, False)"
+ "result_range_query = RangeQuery.SpatialRangeQuery(\n",
+ " linestring_rdd, query_envelope, False, False\n",
+ ")"
]
},
{
@@ -1835,9 +1861,7 @@
],
"source": [
"sedona.createDataFrame(\n",
- " result_range_query.map(lambda x: [x.geom]),\n",
- " schema,\n",
- " verifySchema=False\n",
+ " result_range_query.map(lambda x: [x.geom]), schema, verifySchema=False\n",
").show(5, True)"
]
},
@@ -2325,7 +2349,9 @@
"source": [
"query_envelope = Envelope(-85.01, -60.01, 34.01, 50.01)\n",
"\n",
- "result_range_query = RangeQueryRaw.SpatialRangeQuery(linestring_rdd, query_envelope, False, False)"
+ "result_range_query = RangeQueryRaw.SpatialRangeQuery(\n",
+ " linestring_rdd, query_envelope, False, False\n",
+ ")"
]
},
{
diff --git a/docs/usecases/ApacheSedonaRaster.ipynb b/docs/usecases/ApacheSedonaRaster.ipynb
index 3492f1ae5..042e00963 100644
--- a/docs/usecases/ApacheSedonaRaster.ipynb
+++ b/docs/usecases/ApacheSedonaRaster.ipynb
@@ -149,12 +149,16 @@
}
],
"source": [
- "config = SedonaContext.builder() .\\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,'\n",
- " 'org.datasyslab:geotools-wrapper:1.6.0-28.2,'\n",
- " 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \\\n",
- " getOrCreate()\n",
+ "config = (\n",
+ " SedonaContext.builder()\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.6.0-28.2,\"\n",
+ " \"uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"sedona = SedonaContext.create(config)\n",
"\n",
@@ -360,15 +364,25 @@
}
],
"source": [
- "(width, height) = sedona.sql(\"SELECT RS_Width(raster) as width, RS_Height(raster) as height from raster_table\").first()\n",
- "(p1X, p1Y) = sedona.sql(f\"SELECT RS_RasterToWorldCoordX(raster, {width / 2}, {height / 2}) \\\n",
- " as pX, RS_RasterToWorldCoordY(raster, {width / 2}, {height / 2}) as pY from raster_table\").first()\n",
- "(p2X, p2Y) = sedona.sql(f\"SELECT RS_RasterToWorldCoordX(raster, {(width / 2) + 2}, {height / 2}) \\\n",
- " as pX, RS_RasterToWorldCoordY(raster, {(width / 2) + 2}, {height / 2}) as pY from raster_table\").first()\n",
- "(p3X, p3Y) = sedona.sql(f\"SELECT RS_RasterToWorldCoordX(raster, {width / 2}, {(height / 2) + 2}) \\\n",
- " as pX, RS_RasterToWorldCoordY(raster, {width / 2}, {(height / 2) + 2}) as pY from raster_table\").first()\n",
- "(p4X, p4Y) = sedona.sql(f\"SELECT RS_RasterToWorldCoordX(raster, {(width / 2) + 2}, {(height / 2) + 2}) \\\n",
- " as pX, RS_RasterToWorldCoordY(raster, {(width / 2) + 2}, {(height / 2) + 2}) as pY from raster_table\").first() "
+ "(width, height) = sedona.sql(\n",
+ " \"SELECT RS_Width(raster) as width, RS_Height(raster) as height from raster_table\"\n",
+ ").first()\n",
+ "(p1X, p1Y) = sedona.sql(\n",
+ " f\"SELECT RS_RasterToWorldCoordX(raster, {width / 2}, {height / 2}) \\\n",
+ " as pX, RS_RasterToWorldCoordY(raster, {width / 2}, {height / 2}) as pY from raster_table\"\n",
+ ").first()\n",
+ "(p2X, p2Y) = sedona.sql(\n",
+ " f\"SELECT RS_RasterToWorldCoordX(raster, {(width / 2) + 2}, {height / 2}) \\\n",
+ " as pX, RS_RasterToWorldCoordY(raster, {(width / 2) + 2}, {height / 2}) as pY from raster_table\"\n",
+ ").first()\n",
+ "(p3X, p3Y) = sedona.sql(\n",
+ " f\"SELECT RS_RasterToWorldCoordX(raster, {width / 2}, {(height / 2) + 2}) \\\n",
+ " as pX, RS_RasterToWorldCoordY(raster, {width / 2}, {(height / 2) + 2}) as pY from raster_table\"\n",
+ ").first()\n",
+ "(p4X, p4Y) = sedona.sql(\n",
+ " f\"SELECT RS_RasterToWorldCoordX(raster, {(width / 2) + 2}, {(height / 2) + 2}) \\\n",
+ " as pX, RS_RasterToWorldCoordY(raster, {(width / 2) + 2}, {(height / 2) + 2}) as pY from raster_table\"\n",
+ ").first()"
]
},
{
@@ -418,7 +432,9 @@
}
],
"source": [
- "joined_df = sedona.sql(\"SELECT g.geom from raster_table r, geom_table g where RS_Intersects(r.raster, g.geom)\")\n",
+ "joined_df = sedona.sql(\n",
+ " \"SELECT g.geom from raster_table r, geom_table g where RS_Intersects(r.raster, g.geom)\"\n",
+ ")\n",
"joined_df.show()"
]
},
@@ -490,8 +506,12 @@
}
],
"source": [
- "raster_convex_hull = sedona.sql(\"SELECT RS_ConvexHull(raster) as convex_hull from raster_table\")\n",
- "raster_min_convex_hull = sedona.sql(\"SELECT RS_MinConvexHull(raster) as min_convex_hull from raster_table\")\n",
+ "raster_convex_hull = sedona.sql(\n",
+ " \"SELECT RS_ConvexHull(raster) as convex_hull from raster_table\"\n",
+ ")\n",
+ "raster_min_convex_hull = sedona.sql(\n",
+ " \"SELECT RS_MinConvexHull(raster) as min_convex_hull from raster_table\"\n",
+ ")\n",
"raster_convex_hull.show(truncate=False)\n",
"raster_min_convex_hull.show(truncate=False)"
]
@@ -541,7 +561,9 @@
}
],
"source": [
- "rasterized_geom_df = sedona.sql(\"SELECT RS_AsRaster(ST_GeomFromWKT('POLYGON((150 150, 220 260, 190 300, 300 220, 150 150))'), r.raster, 'b', 230) as rasterized_geom from raster_table r\")\n",
+ "rasterized_geom_df = sedona.sql(\n",
+ " \"SELECT RS_AsRaster(ST_GeomFromWKT('POLYGON((150 150, 220 260, 190 300, 300 220, 150 150))'), r.raster, 'b', 230) as rasterized_geom from raster_table r\"\n",
+ ")\n",
"rasterized_geom_df.show()"
]
},
@@ -585,7 +607,9 @@
}
],
"source": [
- "SedonaUtils.display_image(rasterized_geom_df.selectExpr(\"RS_AsImage(rasterized_geom, 250) as rasterized_geom\"))"
+ "SedonaUtils.display_image(\n",
+ " rasterized_geom_df.selectExpr(\"RS_AsImage(rasterized_geom, 250) as rasterized_geom\")\n",
+ ")"
]
},
{
@@ -611,7 +635,9 @@
},
"outputs": [],
"source": [
- "raster_white_bg = rasterized_geom_df.selectExpr(\"RS_MapAlgebra(rasterized_geom, NULL, 'out[0] = rast[0] == 0 ? 230 : 0;') as raster\")"
+ "raster_white_bg = rasterized_geom_df.selectExpr(\n",
+ " \"RS_MapAlgebra(rasterized_geom, NULL, 'out[0] = rast[0] == 0 ? 230 : 0;') as raster\"\n",
+ ")"
]
},
{
@@ -661,7 +687,9 @@
}
],
"source": [
- "SedonaUtils.display_image(raster_white_bg.selectExpr(\"RS_AsImage(raster, 250) as resampled_raster\"))"
+ "SedonaUtils.display_image(\n",
+ " raster_white_bg.selectExpr(\"RS_AsImage(raster, 250) as resampled_raster\")\n",
+ ")"
]
},
{
@@ -686,7 +714,9 @@
},
"outputs": [],
"source": [
- "resampled_raster_df = sedona.sql(\"SELECT RS_Resample(raster, 1000, 1000, false, 'NearestNeighbor') as resampled_raster from raster_table\")"
+ "resampled_raster_df = sedona.sql(\n",
+ " \"SELECT RS_Resample(raster, 1000, 1000, false, 'NearestNeighbor') as resampled_raster from raster_table\"\n",
+ ")"
]
},
{
@@ -731,7 +761,11 @@
}
],
"source": [
- "SedonaUtils.display_image(resampled_raster_df.selectExpr(\"RS_AsImage(resampled_raster, 500) as resampled_raster\"))"
+ "SedonaUtils.display_image(\n",
+ " resampled_raster_df.selectExpr(\n",
+ " \"RS_AsImage(resampled_raster, 500) as resampled_raster\"\n",
+ " )\n",
+ ")"
]
},
{
@@ -756,7 +790,9 @@
}
],
"source": [
- "resampled_raster_df.selectExpr(\"RS_MetaData(resampled_raster) as resampled_raster_metadata\").show(truncate=False)"
+ "resampled_raster_df.selectExpr(\n",
+ " \"RS_MetaData(resampled_raster) as resampled_raster_metadata\"\n",
+ ").show(truncate=False)"
]
},
{
@@ -769,7 +805,7 @@
"outputs": [],
"source": [
"# Load another raster for some more examples\n",
- "elevation_raster_df = sedona.read.format('binaryFile').load('data/raster/test1.tiff')\n",
+ "elevation_raster_df = sedona.read.format(\"binaryFile\").load(\"data/raster/test1.tiff\")\n",
"elevation_raster_df.createOrReplaceTempView(\"elevation_raster_binary\")"
]
},
@@ -782,7 +818,9 @@
},
"outputs": [],
"source": [
- "elevation_raster_df = sedona.sql(\"SELECT RS_FromGeoTiff(content) as raster from elevation_raster_binary\")\n",
+ "elevation_raster_df = sedona.sql(\n",
+ " \"SELECT RS_FromGeoTiff(content) as raster from elevation_raster_binary\"\n",
+ ")\n",
"elevation_raster_df.createOrReplaceTempView(\"elevation_raster\")"
]
},
@@ -831,11 +869,17 @@
}
],
"source": [
- "point_wkt_1 = 'SRID=3857;POINT (-13095600.809482181 4021100.7487925636)'\n",
- "point_wkt_2 = 'SRID=3857;POINT (-13095500.809482181 4021000.7487925636)'\n",
- "point_df = sedona.sql(\"SELECT ST_GeomFromEWKT('{}') as point_1, ST_GeomFromEWKT('{}') as point_2\".format(point_wkt_1, point_wkt_2))\n",
+ "point_wkt_1 = \"SRID=3857;POINT (-13095600.809482181 4021100.7487925636)\"\n",
+ "point_wkt_2 = \"SRID=3857;POINT (-13095500.809482181 4021000.7487925636)\"\n",
+ "point_df = sedona.sql(\n",
+ " \"SELECT ST_GeomFromEWKT('{}') as point_1, ST_GeomFromEWKT('{}') as point_2\".format(\n",
+ " point_wkt_1, point_wkt_2\n",
+ " )\n",
+ ")\n",
"point_df.createOrReplaceTempView(\"point_table\")\n",
- "test_df = sedona.sql(\"SELECT RS_Values(raster, Array(point_1, point_2)) as raster_values from elevation_raster, point_table\")\n",
+ "test_df = sedona.sql(\n",
+ " \"SELECT RS_Values(raster, Array(point_1, point_2)) as raster_values from elevation_raster, point_table\"\n",
+ ")\n",
"test_df.show()"
]
},
@@ -864,7 +908,9 @@
],
"source": [
"band = elevation_raster_df.selectExpr(\"RS_BandAsArray(raster, 1)\").first()[0]\n",
- "print(band[500: 520],) #Print a part of a band as an array horizontally"
+ "print(\n",
+ " band[500:520],\n",
+ ") # Print a part of a band as an array horizontally"
]
},
{
@@ -883,7 +929,9 @@
"outputs": [],
"source": [
"# Convert raster to its convex hull and transform it to EPSG:4326 to be able to visualize\n",
- "raster_mbr_df = elevation_raster_df.selectExpr(\"ST_Transform(RS_ConvexHull(raster), 'EPSG:3857', 'EPSG:4326') as raster_mbr\")"
+ "raster_mbr_df = elevation_raster_df.selectExpr(\n",
+ " \"ST_Transform(RS_ConvexHull(raster), 'EPSG:3857', 'EPSG:4326') as raster_mbr\"\n",
+ ")"
]
},
{
@@ -895,7 +943,9 @@
},
"outputs": [],
"source": [
- "sedona_kepler_map_elevation = SedonaKepler.create_map(df=raster_mbr_df, name='RasterMBR')\n",
+ "sedona_kepler_map_elevation = SedonaKepler.create_map(\n",
+ " df=raster_mbr_df, name=\"RasterMBR\"\n",
+ ")\n",
"sedona_kepler_map_elevation"
]
},
diff --git a/docs/usecases/ApacheSedonaSQL.ipynb b/docs/usecases/ApacheSedonaSQL.ipynb
index 5d4ccebf9..f36f1ec60 100644
--- a/docs/usecases/ApacheSedonaSQL.ipynb
+++ b/docs/usecases/ApacheSedonaSQL.ipynb
@@ -133,14 +133,18 @@
}
],
"source": [
- "config = SedonaContext.builder() .\\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,'\n",
- " 'org.datasyslab:geotools-wrapper:1.6.0-28.2,'\n",
- " 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \\\n",
- " getOrCreate()\n",
+ "config = (\n",
+ " SedonaContext.builder()\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.6.0-28.2,\"\n",
+ " \"uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
- "sedona = SedonaContext.create(config)\n"
+ "sedona = SedonaContext.create(config)"
]
},
{
@@ -195,14 +199,18 @@
}
],
"source": [
- "point_csv_df = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \",\").\\\n",
- " option(\"header\", \"false\").\\\n",
- " load(\"data/testpoint.csv\")\n",
+ "point_csv_df = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \",\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/testpoint.csv\")\n",
+ ")\n",
"\n",
"point_csv_df.createOrReplaceTempView(\"pointtable\")\n",
"\n",
- "point_df = sedona.sql(\"select ST_Point(cast(pointtable._c0 as Decimal(24,20)), cast(pointtable._c1 as Decimal(24,20))) as arealandmark from pointtable\")\n",
+ "point_df = sedona.sql(\n",
+ " \"select ST_Point(cast(pointtable._c0 as Decimal(24,20)), cast(pointtable._c1 as Decimal(24,20))) as arealandmark from pointtable\"\n",
+ ")\n",
"point_df.show(5)"
]
},
@@ -237,13 +245,17 @@
}
],
"source": [
- "polygon_wkt_df = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \"\\t\").\\\n",
- " option(\"header\", \"false\").\\\n",
- " load(\"data/county_small.tsv\")\n",
+ "polygon_wkt_df = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \"\\t\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/county_small.tsv\")\n",
+ ")\n",
"\n",
"polygon_wkt_df.createOrReplaceTempView(\"polygontable\")\n",
- "polygon_df = sedona.sql(\"select polygontable._c6 as name, ST_GeomFromText(polygontable._c0) as countyshape from polygontable\")\n",
+ "polygon_df = sedona.sql(\n",
+ " \"select polygontable._c6 as name, ST_GeomFromText(polygontable._c0) as countyshape from polygontable\"\n",
+ ")\n",
"polygon_df.show(5)"
]
},
@@ -278,13 +290,17 @@
}
],
"source": [
- "polygon_wkb_df = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \"\\t\").\\\n",
- " option(\"header\", \"false\").\\\n",
- " load(\"data/county_small_wkb.tsv\")\n",
+ "polygon_wkb_df = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \"\\t\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/county_small_wkb.tsv\")\n",
+ ")\n",
"\n",
"polygon_wkb_df.createOrReplaceTempView(\"polygontable\")\n",
- "polygon_df = sedona.sql(\"select polygontable._c6 as name, ST_GeomFromWKB(polygontable._c0) as countyshape from polygontable\")\n",
+ "polygon_df = sedona.sql(\n",
+ " \"select polygontable._c6 as name, ST_GeomFromWKB(polygontable._c0) as countyshape from polygontable\"\n",
+ ")\n",
"polygon_df.show(5)"
]
},
@@ -319,13 +335,17 @@
}
],
"source": [
- "polygon_json_df = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \"\\t\").\\\n",
- " option(\"header\", \"false\").\\\n",
- " load(\"data/testPolygon.json\")\n",
+ "polygon_json_df = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \"\\t\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/testPolygon.json\")\n",
+ ")\n",
"\n",
"polygon_json_df.createOrReplaceTempView(\"polygontable\")\n",
- "polygon_df = sedona.sql(\"select ST_GeomFromGeoJSON(polygontable._c0) as countyshape from polygontable\")\n",
+ "polygon_df = sedona.sql(\n",
+ " \"select ST_GeomFromGeoJSON(polygontable._c0) as countyshape from polygontable\"\n",
+ ")\n",
"polygon_df.show(5)"
]
},
@@ -389,24 +409,36 @@
}
],
"source": [
- "point_csv_df_1 = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \",\").\\\n",
- " option(\"header\", \"false\").load(\"data/testpoint.csv\")\n",
+ "point_csv_df_1 = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \",\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/testpoint.csv\")\n",
+ ")\n",
"\n",
"point_csv_df_1.createOrReplaceTempView(\"pointtable\")\n",
"\n",
- "point_df1 = sedona.sql(\"SELECT ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape1, \\'abc\\' as name1 from pointtable\")\n",
+ "point_df1 = sedona.sql(\n",
+ " \"SELECT ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape1, 'abc' as name1 from pointtable\"\n",
+ ")\n",
"point_df1.createOrReplaceTempView(\"pointdf1\")\n",
"\n",
- "point_csv_df2 = sedona.read.format(\"csv\").\\\n",
- " option(\"delimiter\", \",\").\\\n",
- " option(\"header\", \"false\").load(\"data/testpoint.csv\")\n",
+ "point_csv_df2 = (\n",
+ " sedona.read.format(\"csv\")\n",
+ " .option(\"delimiter\", \",\")\n",
+ " .option(\"header\", \"false\")\n",
+ " .load(\"data/testpoint.csv\")\n",
+ ")\n",
"\n",
"point_csv_df2.createOrReplaceTempView(\"pointtable\")\n",
- "point_df2 = sedona.sql(\"select ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape2, \\'def\\' as name2 from pointtable\")\n",
+ "point_df2 = sedona.sql(\n",
+ " \"select ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape2, 'def' as name2 from pointtable\"\n",
+ ")\n",
"point_df2.createOrReplaceTempView(\"pointdf2\")\n",
"\n",
- "distance_join_df = sedona.sql(\"select * from pointdf1, pointdf2 where ST_Distance(pointdf1.pointshape1,pointdf2.pointshape2) < 2\")\n",
+ "distance_join_df = sedona.sql(\n",
+ " \"select * from pointdf1, pointdf2 where ST_Distance(pointdf1.pointshape1,pointdf2.pointshape2) < 2\"\n",
+ ")\n",
"distance_join_df.explain()\n",
"distance_join_df.show(5)"
]
@@ -434,11 +466,10 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "\n",
"gdf = gpd.read_file(\"data/gis_osm_pois_free_1.shp\")\n",
- "gdf = gdf.replace(pd.NA, '')\n",
- "osm_points = sedona.createDataFrame(\n",
- " gdf\n",
- ")"
+ "gdf = gdf.replace(pd.NA, \"\")\n",
+ "osm_points = sedona.createDataFrame(gdf)"
]
},
{
@@ -528,7 +559,8 @@
" name,\n",
" ST_Transform(geometry, 'epsg:4326', 'epsg:2180') as geom \n",
" FROM points\n",
- " \"\"\")"
+ " \"\"\"\n",
+ ")"
]
},
{
@@ -587,13 +619,15 @@
"metadata": {},
"outputs": [],
"source": [
- "neighbours_within_1000m = sedona.sql(\"\"\"\n",
+ "neighbours_within_1000m = sedona.sql(\n",
+ " \"\"\"\n",
" SELECT a.osm_id AS id_1,\n",
" b.osm_id AS id_2,\n",
" a.geom \n",
" FROM points_2180 AS a, points_2180 AS b \n",
" WHERE ST_Distance(a.geom,b.geom) < 50\n",
- " \"\"\")"
+ " \"\"\"\n",
+ ")"
]
},
{
diff --git a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
index 97eeb96ce..8883b98f8 100644
--- a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
+++ b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
@@ -35,7 +35,6 @@
"from pyspark.sql.functions import col, expr, when, explode, hex\n",
"\n",
"\n",
- "\n",
"from sedona.spark import *\n",
"from utilities import getConfig"
]
@@ -99,12 +98,16 @@
}
],
"source": [
- "config = SedonaContext.builder() .\\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-spark-shaded-3.4_2.12:1.6.0,'\n",
- " 'org.datasyslab:geotools-wrapper:1.6.0-28.2,'\n",
- " 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \\\n",
- " getOrCreate()\n",
+ "config = (\n",
+ " SedonaContext.builder()\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-spark-shaded-3.4_2.12:1.6.0,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.6.0-28.2,\"\n",
+ " \"uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"sedona = SedonaContext.create(config)\n",
"sc = sedona.sparkContext\n",
@@ -236,7 +239,9 @@
}
],
"source": [
- "countries = ShapefileReader.readToGeometryRDD(sc, \"data/ne_50m_admin_0_countries_lakes/\")\n",
+ "countries = ShapefileReader.readToGeometryRDD(\n",
+ " sc, \"data/ne_50m_admin_0_countries_lakes/\"\n",
+ ")\n",
"countries_df = Adapter.toDf(countries, sedona)\n",
"countries_df.createOrReplaceTempView(\"country\")\n",
"countries_df.printSchema()"
@@ -297,7 +302,9 @@
"metadata": {},
"outputs": [],
"source": [
- "result = sedona.sql(\"SELECT c.geometry as country_geom, c.NAME_EN, a.geometry as airport_geom, a.name FROM country c, airport a WHERE ST_Contains(c.geometry, a.geometry)\")"
+ "result = sedona.sql(\n",
+ " \"SELECT c.geometry as country_geom, c.NAME_EN, a.geometry as airport_geom, a.name FROM country c, airport a WHERE ST_Contains(c.geometry, a.geometry)\"\n",
+ ")"
]
},
{
@@ -338,13 +345,19 @@
"considerBoundaryIntersection = True\n",
"airports_rdd.buildIndex(IndexType.QUADTREE, buildOnSpatialPartitionedRDD)\n",
"\n",
- "result_pair_rdd = JoinQueryRaw.SpatialJoinQueryFlat(airports_rdd, countries_rdd, usingIndex, considerBoundaryIntersection)\n",
+ "result_pair_rdd = JoinQueryRaw.SpatialJoinQueryFlat(\n",
+ " airports_rdd, countries_rdd, usingIndex, considerBoundaryIntersection\n",
+ ")\n",
"\n",
- "result2 = Adapter.toDf(result_pair_rdd, countries_rdd.fieldNames, airports.fieldNames, sedona)\n",
+ "result2 = Adapter.toDf(\n",
+ " result_pair_rdd, countries_rdd.fieldNames, airports.fieldNames, sedona\n",
+ ")\n",
"\n",
"result2.createOrReplaceTempView(\"join_result_with_all_cols\")\n",
"# Select the columns needed in the join\n",
- "result2 = sedona.sql(\"SELECT leftgeometry as country_geom, NAME_EN, rightgeometry as airport_geom, name FROM join_result_with_all_cols\")"
+ "result2 = sedona.sql(\n",
+ " \"SELECT leftgeometry as country_geom, NAME_EN, rightgeometry as airport_geom, name FROM join_result_with_all_cols\"\n",
+ ")"
]
},
{
@@ -503,7 +516,9 @@
"source": [
"# result.createOrReplaceTempView(\"result\")\n",
"result2.createOrReplaceTempView(\"result\")\n",
- "groupedresult = sedona.sql(\"SELECT c.NAME_EN, c.country_geom, count(*) as AirportCount FROM result c GROUP BY c.NAME_EN, c.country_geom\")\n",
+ "groupedresult = sedona.sql(\n",
+ " \"SELECT c.NAME_EN, c.country_geom, count(*) as AirportCount FROM result c GROUP BY c.NAME_EN, c.country_geom\"\n",
+ ")\n",
"groupedresult.show()\n",
"groupedresult.createOrReplaceTempView(\"grouped_result\")"
]
@@ -557,7 +572,9 @@
}
],
"source": [
- "sedona_kepler_map = SedonaKepler.create_map(df=groupedresult, name=\"AirportCount\", config=getConfig())\n",
+ "sedona_kepler_map = SedonaKepler.create_map(\n",
+ " df=groupedresult, name=\"AirportCount\", config=getConfig()\n",
+ ")\n",
"sedona_kepler_map"
]
},
@@ -669373,7 +669390,9 @@
}
],
"source": [
- "sedona_pydeck_map = SedonaPyDeck.create_choropleth_map(df=groupedresult, plot_col='AirportCount')\n",
+ "sedona_pydeck_map = SedonaPyDeck.create_choropleth_map(\n",
+ " df=groupedresult, plot_col=\"AirportCount\"\n",
+ ")\n",
"sedona_pydeck_map"
]
},
@@ -669430,7 +669449,9 @@
}
],
"source": [
- "h3_df = sedona.sql(\"SELECT g.NAME_EN, g.country_geom, ST_H3CellIDs(g.country_geom, 3, false) as h3_cellID from grouped_result g\")\n",
+ "h3_df = sedona.sql(\n",
+ " \"SELECT g.NAME_EN, g.country_geom, ST_H3CellIDs(g.country_geom, 3, false) as h3_cellID from grouped_result g\"\n",
+ ")\n",
"h3_df.show(2)"
]
},
@@ -669478,7 +669499,9 @@
}
],
"source": [
- "exploded_h3 = h3_df.select(h3_df.NAME_EN, h3_df.country_geom, explode(h3_df.h3_cellID).alias(\"h3\"))\n",
+ "exploded_h3 = h3_df.select(\n",
+ " h3_df.NAME_EN, h3_df.country_geom, explode(h3_df.h3_cellID).alias(\"h3\")\n",
+ ")\n",
"exploded_h3.show(2)"
]
},
@@ -669533,7 +669556,9 @@
"source": [
"exploded_h3 = exploded_h3.sample(0.3)\n",
"exploded_h3.createOrReplaceTempView(\"exploded_h3\")\n",
- "hex_exploded_h3 = exploded_h3.select(exploded_h3.NAME_EN, hex(exploded_h3.h3).alias(\"ex_h3\"))\n",
+ "hex_exploded_h3 = exploded_h3.select(\n",
+ " exploded_h3.NAME_EN, hex(exploded_h3.h3).alias(\"ex_h3\")\n",
+ ")\n",
"hex_exploded_h3.show(2)\n",
"hex_exploded_h3.printSchema()"
]
diff --git a/docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb b/docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb
index 17389d040..875700faf 100644
--- a/docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb
+++ b/docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb
@@ -70,7 +70,9 @@
},
"outputs": [],
"source": [
- "DATA_LINK = \"s3a://wherobots-examples/data/overturemaps-us-west-2/release/2023-07-26-alpha.0/\""
+ "DATA_LINK = (\n",
+ " \"s3a://wherobots-examples/data/overturemaps-us-west-2/release/2023-07-26-alpha.0/\"\n",
+ ")"
]
},
{
@@ -212,14 +214,24 @@
}
],
"source": [
- "config = SedonaContext.builder() .\\\n",
- " config(\"spark.hadoop.fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider\"). \\\n",
- " config(\"fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider\"). \\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,'\n",
- " 'org.datasyslab:geotools-wrapper:1.6.0-28.2,'\n",
- " 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \\\n",
- " getOrCreate()\n",
+ "config = (\n",
+ " SedonaContext.builder()\n",
+ " .config(\n",
+ " \"spark.hadoop.fs.s3a.aws.credentials.provider\",\n",
+ " \"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider\",\n",
+ " )\n",
+ " .config(\n",
+ " \"fs.s3a.aws.credentials.provider\",\n",
+ " \"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider\",\n",
+ " )\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.6.0-28.2,\"\n",
+ " \"uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"sedona = SedonaContext.create(config)"
]
@@ -254,7 +266,7 @@
"outputs": [],
"source": [
"# Washington state boundary\n",
- "#spatial_filter = \"POLYGON((-123.3208 49.0023,-123.0338 49.0027,-122.0650 49.0018,-121.7491 48.9973,-121.5912 48.9991,-119.6082 49.0009,-118.0378 49.0005,-117.0319 48.9996,-117.0415 47.9614,-117.0394 46.5060,-117.0394 46.4274,-117.0621 46.3498,-117.0277 46.3384,-116.9879 46.2848,-116.9577 46.2388,-116.9659 46.2022,-116.9254 46.1722,-116.9357 46.1432,-116.9584 46.1009,-116.9762 46.0785,-116.9433 46.0537,-116.9165 45.9960,-118.0330 46.0008,-118.9867 45.9998,-119.1302 45.9320,-119.170 [...]
+ "# spatial_filter = \"POLYGON((-123.3208 49.0023,-123.0338 49.0027,-122.0650 49.0018,-121.7491 48.9973,-121.5912 48.9991,-119.6082 49.0009,-118.0378 49.0005,-117.0319 48.9996,-117.0415 47.9614,-117.0394 46.5060,-117.0394 46.4274,-117.0621 46.3498,-117.0277 46.3384,-116.9879 46.2848,-116.9577 46.2388,-116.9659 46.2022,-116.9254 46.1722,-116.9357 46.1432,-116.9584 46.1009,-116.9762 46.0785,-116.9433 46.0537,-116.9165 45.9960,-118.0330 46.0008,-118.9867 45.9998,-119.1302 45.9320,-119.17 [...]
"\n",
"# Bellevue city boundary\n",
"spatial_filter = \"POLYGON ((-122.235128 47.650163, -122.233796 47.65162, -122.231581 47.653287, -122.228514 47.65482, -122.227526 47.655204, -122.226175 47.655729, -122.222039 47.656743999999996, -122.218428 47.657464, -122.217026 47.657506, -122.21437399999999 47.657588, -122.212091 47.657464, -122.212135 47.657320999999996, -122.21092999999999 47.653552, -122.209834 47.650121, -122.209559 47.648976, -122.209642 47.648886, -122.21042 47.648658999999995, -122.210897 47.64864, -122. [...]
@@ -376,7 +388,7 @@
}
],
"source": [
- "sedona.read.parquet_blocks(DATA_LINK+\"theme=places/type=place\").show()"
+ "sedona.read.parquet_blocks(DATA_LINK + \"theme=places/type=place\").show()"
]
},
{
@@ -426,7 +438,9 @@
}
],
"source": [
- "sedona.read.format(\"geoparquet.metadata\").load(DATA_LINK+\"theme=places/type=place\").drop(\"path\").printSchema()"
+ "sedona.read.format(\"geoparquet.metadata\").load(\n",
+ " DATA_LINK + \"theme=places/type=place\"\n",
+ ").drop(\"path\").printSchema()"
]
},
{
@@ -485,7 +499,9 @@
}
],
"source": [
- "sedona.read.format(\"geoparquet.metadata\").load(DATA_LINK+\"theme=places/type=place\").drop(\"path\").show(truncate = False)"
+ "sedona.read.format(\"geoparquet.metadata\").load(\n",
+ " DATA_LINK + \"theme=places/type=place\"\n",
+ ").drop(\"path\").show(truncate=False)"
]
},
{
@@ -573,7 +589,9 @@
}
],
"source": [
- "sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=places/type=place\").printSchema()"
+ "sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=places/type=place\"\n",
+ ").printSchema()"
]
},
{
@@ -613,9 +631,11 @@
"source": [
"%%time\n",
"\n",
- "df_place = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=places/type=place\")\n",
+ "df_place = sedona.read.format(\"geoparquet\").load(DATA_LINK + \"theme=places/type=place\")\n",
"\n",
- "df_place = df_place.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\").cache()"
+ "df_place = df_place.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ").cache()"
]
},
{
@@ -698,8 +718,15 @@
}
],
"source": [
- "df_place.select(\"id\", \"geometry\", \"categories.main\").limit(1000).repartition(1) \\\n",
- " .write.format(\"geoparquet\").option(\"geoparquet.version\", \"1.0.0\").option(\"geoparquet.crs\", \"\").mode('overwrite').save(\"places.parquet\")"
+ "df_place.select(\"id\", \"geometry\", \"categories.main\").limit(1000).repartition(\n",
+ " 1\n",
+ ").write.format(\"geoparquet\").option(\"geoparquet.version\", \"1.0.0\").option(\n",
+ " \"geoparquet.crs\", \"\"\n",
+ ").mode(\n",
+ " \"overwrite\"\n",
+ ").save(\n",
+ " \"places.parquet\"\n",
+ ")"
]
},
{
@@ -861,9 +888,12 @@
}
],
"source": [
- "gdf = gpd.GeoDataFrame(df_place.select(\"id\", \"geometry\", \"categories.main\").limit(1000).toPandas(), geometry=\"geometry\")\n",
- "gdf.to_file('places.geojson', driver='GeoJSON')\n",
- "gdf.to_file('places.shp')\n",
+ "gdf = gpd.GeoDataFrame(\n",
+ " df_place.select(\"id\", \"geometry\", \"categories.main\").limit(1000).toPandas(),\n",
+ " geometry=\"geometry\",\n",
+ ")\n",
+ "gdf.to_file(\"places.geojson\", driver=\"GeoJSON\")\n",
+ "gdf.to_file(\"places.shp\")\n",
"gdf"
]
},
@@ -904,9 +934,13 @@
"source": [
"%%time\n",
"\n",
- "df_building = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=buildings/type=building\")\n",
+ "df_building = sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=buildings/type=building\"\n",
+ ")\n",
"\n",
- "df_building = df_building.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\")\n",
+ "df_building = df_building.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ")\n",
"\n",
"df_building = df_building.limit(200_000)"
]
@@ -962,7 +996,7 @@
"source": [
"%%time\n",
"\n",
- "map_building = SedonaKepler.create_map(df_building, 'Building')\n",
+ "map_building = SedonaKepler.create_map(df_building, \"Building\")\n",
"map_building"
]
},
@@ -1007,9 +1041,13 @@
"source": [
"%%time\n",
"\n",
- "df_admin = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=admins/type=administrativeBoundary\")\n",
+ "df_admin = sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=admins/type=administrativeBoundary\"\n",
+ ")\n",
"\n",
- "df_admin = df_admin.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\")"
+ "df_admin = df_admin.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ")"
]
},
{
@@ -1103,9 +1141,13 @@
"source": [
"%%time\n",
"\n",
- "df_locality = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=admins/type=locality\")\n",
+ "df_locality = sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=admins/type=locality\"\n",
+ ")\n",
"\n",
- "df_locality = df_locality.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\")"
+ "df_locality = df_locality.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ")"
]
},
{
@@ -1161,7 +1203,7 @@
"source": [
"%%time\n",
"\n",
- "map_locality = SedonaKepler.create_map(df_locality, 'Locality')\n",
+ "map_locality = SedonaKepler.create_map(df_locality, \"Locality\")\n",
"\n",
"map_locality"
]
@@ -1207,9 +1249,13 @@
"source": [
"%%time\n",
"\n",
- "df_connector = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=transportation/type=connector\")\n",
+ "df_connector = sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=transportation/type=connector\"\n",
+ ")\n",
"\n",
- "df_connector = df_connector.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\")"
+ "df_connector = df_connector.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ")"
]
},
{
@@ -1306,9 +1352,13 @@
"source": [
"%%time\n",
"\n",
- "df_segment = sedona.read.format(\"geoparquet\").load(DATA_LINK+\"theme=transportation/type=segment\")\n",
+ "df_segment = sedona.read.format(\"geoparquet\").load(\n",
+ " DATA_LINK + \"theme=transportation/type=segment\"\n",
+ ")\n",
"\n",
- "df_segment = df_segment.filter(\"ST_Contains(ST_GeomFromWKT('\"+spatial_filter+\"'), geometry) = true\")\n",
+ "df_segment = df_segment.filter(\n",
+ " \"ST_Contains(ST_GeomFromWKT('\" + spatial_filter + \"'), geometry) = true\"\n",
+ ")\n",
"\n",
"df_segment = df_segment.limit(200000)"
]
diff --git a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
index 8dea56b16..13d4ea7f7 100644
--- a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
+++ b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb
@@ -52,7 +52,15 @@
"from pyspark.sql import SparkSession\n",
"from pyspark import StorageLevel\n",
"import pandas as pd\n",
- "from pyspark.sql.types import StructType, StructField,StringType, LongType, IntegerType, DoubleType, ArrayType\n",
+ "from pyspark.sql.types import (\n",
+ " StructType,\n",
+ " StructField,\n",
+ " StringType,\n",
+ " LongType,\n",
+ " IntegerType,\n",
+ " DoubleType,\n",
+ " ArrayType,\n",
+ ")\n",
"from pyspark.sql.functions import regexp_replace\n",
"from sedona.register import SedonaRegistrator\n",
"from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
@@ -120,18 +128,21 @@
}
],
"source": [
- "spark = SparkSession.\\\n",
- " builder.\\\n",
- " appName(\"Demo-app\").\\\n",
- " enableHiveSupport().\\\n",
- " master(\"local[*]\").\\\n",
- " master(\"spark://spark-master:7077\").\\\n",
- " config(\"spark.executor.memory\", \"15G\").\\\n",
- " config(\"spark.driver.maxResultSize\", \"15G\").\\\n",
- " config(\"spark.serializer\", KryoSerializer.getName).\\\n",
- " config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName).\\\n",
- " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\") .\\\n",
- " getOrCreate()\n",
+ "spark = (\n",
+ " SparkSession.builder.appName(\"Demo-app\")\n",
+ " .enableHiveSupport()\n",
+ " .master(\"local[*]\")\n",
+ " .master(\"spark://spark-master:7077\")\n",
+ " .config(\"spark.executor.memory\", \"15G\")\n",
+ " .config(\"spark.driver.maxResultSize\", \"15G\")\n",
+ " .config(\"spark.serializer\", KryoSerializer.getName)\n",
+ " .config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")\n",
"# config(\"spark.rpc.message.maxSize\", 2047).\\\n",
"# rdd = spark.sparkContext.parallelize(range(1000))\n",
"# rdd.takeSample(False, 5)\n",
@@ -147,9 +158,9 @@
"metadata": {},
"outputs": [],
"source": [
- "# Path to directory of geotiff images \n",
+ "# Path to directory of geotiff images\n",
"DATA_DIR = \"hdfs://776faf4d6a1e:8020/tmp/\"\n",
- "df = spark.read.format(\"geotiff\").option(\"dropInvalid\",True).load(DATA_DIR)"
+ "df = spark.read.format(\"geotiff\").option(\"dropInvalid\", True).load(DATA_DIR)"
]
},
{
@@ -218,7 +229,14 @@
"source": [
"# Java Heap Out Of Memory => Ir nas máquinas e aumentar o export _JAVA_OPTIONS=\"-Xmx15g\"\n",
"# Java lang Assertion Error image is too large =>\n",
- "df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\").cache()\n",
+ "df = df.selectExpr(\n",
+ " \"image.origin as origin\",\n",
+ " \"ST_GeomFromWkt(image.wkt) as Geom\",\n",
+ " \"image.height as height\",\n",
+ " \"image.width as width\",\n",
+ " \"image.data as data\",\n",
+ " \"image.nBands as bands\",\n",
+ ").cache()\n",
"df.show(5)\n",
"# df.count()"
]
@@ -231,7 +249,13 @@
"outputs": [],
"source": [
"# ,\"RS_GetBand(data, 2,bands) as Band2\",\"RS_GetBand(data, 3,bands) as Band3\", \"RS_GetBand(data, 4,bands) as Band4\"\n",
- "df = df.selectExpr(\"Geom\",\"RS_GetBand(data, 1,bands) as Band1\",\"RS_GetBand(data, 2,bands) as Band2\",\"RS_GetBand(data, 3,bands) as Band3\", \"RS_GetBand(data, 4,bands) as Band4\").cache()\n",
+ "df = df.selectExpr(\n",
+ " \"Geom\",\n",
+ " \"RS_GetBand(data, 1,bands) as Band1\",\n",
+ " \"RS_GetBand(data, 2,bands) as Band2\",\n",
+ " \"RS_GetBand(data, 3,bands) as Band3\",\n",
+ " \"RS_GetBand(data, 4,bands) as Band4\",\n",
+ ").cache()\n",
"df.createOrReplaceTempView(\"allbands\")\n",
"df.show(5)"
]
@@ -254,7 +278,9 @@
"metadata": {},
"outputs": [],
"source": [
- "NomalizedDifference = df.selectExpr(\"RS_NormalizedDifference(Band1, Band2) as normDiff\").cache()\n",
+ "NomalizedDifference = df.selectExpr(\n",
+ " \"RS_NormalizedDifference(Band1, Band2) as normDiff\"\n",
+ ").cache()\n",
"NomalizedDifference.show(5)"
]
},
@@ -287,7 +313,9 @@
"metadata": {},
"outputs": [],
"source": [
- "greaterthanDF = spark.sql(\"Select RS_GreaterThan(Band1,1000.0) as greaterthan from allbands\").cache()\n",
+ "greaterthanDF = spark.sql(\n",
+ " \"Select RS_GreaterThan(Band1,1000.0) as greaterthan from allbands\"\n",
+ ").cache()\n",
"greaterthanDF.show()"
]
},
@@ -298,7 +326,9 @@
"metadata": {},
"outputs": [],
"source": [
- "greaterthanEqualDF = spark.sql(\"Select RS_GreaterThanEqual(Band1,360.0) as greaterthanEqual from allbands\").cache()\n",
+ "greaterthanEqualDF = spark.sql(\n",
+ " \"Select RS_GreaterThanEqual(Band1,360.0) as greaterthanEqual from allbands\"\n",
+ ").cache()\n",
"greaterthanEqualDF.show()"
]
},
@@ -309,7 +339,9 @@
"metadata": {},
"outputs": [],
"source": [
- "lessthanDF = spark.sql(\"Select RS_LessThan(Band1,1000.0) as lessthan from allbands\").cache()\n",
+ "lessthanDF = spark.sql(\n",
+ " \"Select RS_LessThan(Band1,1000.0) as lessthan from allbands\"\n",
+ ").cache()\n",
"lessthanDF.show()"
]
},
@@ -320,7 +352,9 @@
"metadata": {},
"outputs": [],
"source": [
- "lessthanEqualDF = spark.sql(\"Select RS_LessThanEqual(Band1,2890.0) as lessthanequal from allbands\").cache()\n",
+ "lessthanEqualDF = spark.sql(\n",
+ " \"Select RS_LessThanEqual(Band1,2890.0) as lessthanequal from allbands\"\n",
+ ").cache()\n",
"lessthanEqualDF.show()"
]
},
@@ -463,12 +497,33 @@
"metadata": {},
"outputs": [],
"source": [
- "df = spark.read.format(\"geotiff\").option(\"dropInvalid\",True).load(DATA_DIR)\n",
- "df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\").cache()\n",
+ "df = spark.read.format(\"geotiff\").option(\"dropInvalid\", True).load(DATA_DIR)\n",
+ "df = df.selectExpr(\n",
+ " \"image.origin as origin\",\n",
+ " \"ST_GeomFromWkt(image.wkt) as Geom\",\n",
+ " \"image.height as height\",\n",
+ " \"image.width as width\",\n",
+ " \"image.data as data\",\n",
+ " \"image.nBands as bands\",\n",
+ ").cache()\n",
"\n",
- "df = df.selectExpr(\"RS_GetBand(data,1,bands) as targetband\", \"height\", \"width\", \"bands\", \"Geom\")\n",
- "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Array(height*width,0.0), RS_Array(height*width, 0.0)) as red\",\"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Normalize(targetBand), RS_Array(height*width, 0.0)) as green\", \"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Array(height*width, 0.0), RS_Normalize(targetBand)) as blue\",\"RS_Base64(height,width,RS_Normalize(targetBand), RS_Normalize(targetBand),RS_Normalize(targ [...]
- "df_HTML = df_base64.selectExpr(\"Geom\",\"RS_HTML(red) as RedBand\",\"RS_HTML(blue) as BlueBand\",\"RS_HTML(green) as GreenBand\", \"RS_HTML(RGB) as CombinedBand\").cache()\n",
+ "df = df.selectExpr(\n",
+ " \"RS_GetBand(data,1,bands) as targetband\", \"height\", \"width\", \"bands\", \"Geom\"\n",
+ ")\n",
+ "df_base64 = df.selectExpr(\n",
+ " \"Geom\",\n",
+ " \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Array(height*width,0.0), RS_Array(height*width, 0.0)) as red\",\n",
+ " \"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Normalize(targetBand), RS_Array(height*width, 0.0)) as green\",\n",
+ " \"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Array(height*width, 0.0), RS_Normalize(targetBand)) as blue\",\n",
+ " \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Normalize(targetBand),RS_Normalize(targetBand)) as RGB\",\n",
+ ").cache()\n",
+ "df_HTML = df_base64.selectExpr(\n",
+ " \"Geom\",\n",
+ " \"RS_HTML(red) as RedBand\",\n",
+ " \"RS_HTML(blue) as BlueBand\",\n",
+ " \"RS_HTML(green) as GreenBand\",\n",
+ " \"RS_HTML(RGB) as CombinedBand\",\n",
+ ").cache()\n",
"df_HTML.show(5)"
]
},
@@ -489,14 +544,14 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"def SumOfValues(band):\n",
" total = 0.0\n",
" for num in band:\n",
- " if num>1000.0:\n",
- " total+=1\n",
+ " if num > 1000.0:\n",
+ " total += 1\n",
" return total\n",
- " \n",
+ "\n",
+ "\n",
"calculateSum = udf(SumOfValues, DoubleType())\n",
"spark.udf.register(\"RS_Sum\", calculateSum)\n",
"\n",
@@ -511,20 +566,26 @@
"metadata": {},
"outputs": [],
"source": [
- "def generatemask(band, width,height):\n",
- " for (i,val) in enumerate(band):\n",
- " if (i%width>=12 and i%width<26) and (i%height>=12 and i%height<26):\n",
+ "def generatemask(band, width, height):\n",
+ " for i, val in enumerate(band):\n",
+ " if (i % width >= 12 and i % width < 26) and (\n",
+ " i % height >= 12 and i % height < 26\n",
+ " ):\n",
" band[i] = 255.0\n",
" else:\n",
" band[i] = 0.0\n",
" return band\n",
"\n",
+ "\n",
"maskValues = udf(generatemask, ArrayType(DoubleType()))\n",
"spark.udf.register(\"RS_MaskValues\", maskValues)\n",
"\n",
"\n",
- "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetband), RS_Array(height*width,0.0), RS_Array(height*width, 0.0), RS_MaskValues(targetband,width,height)) as region\" ).cache()\n",
- "df_HTML = df_base64.selectExpr(\"Geom\",\"RS_HTML(region) as selectedregion\").cache()\n",
+ "df_base64 = df.selectExpr(\n",
+ " \"Geom\",\n",
+ " \"RS_Base64(height,width,RS_Normalize(targetband), RS_Array(height*width,0.0), RS_Array(height*width, 0.0), RS_MaskValues(targetband,width,height)) as region\",\n",
+ ").cache()\n",
+ "df_HTML = df_base64.selectExpr(\"Geom\", \"RS_HTML(region) as selectedregion\").cache()\n",
"display(HTML(df_HTML.limit(2).toPandas().to_html(escape=False)))"
]
},
diff --git a/docs/usecases/contrib/DownloadImageFromGEE.ipynb b/docs/usecases/contrib/DownloadImageFromGEE.ipynb
index 9d5a873c0..b90b63047 100644
--- a/docs/usecases/contrib/DownloadImageFromGEE.ipynb
+++ b/docs/usecases/contrib/DownloadImageFromGEE.ipynb
@@ -119,11 +119,19 @@
"percent_cloud = 100\n",
"# box = [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]]\n",
"# https://boundingbox.klokantech.com/\n",
- "box = [[[-54.6306579887,-25.5892766534],[-54.5393341362,-25.5892766534],[-54.5393341362,-25.4046299874],[-54.6306579887,-25.4046299874],[-54.6306579887,-25.5892766534]]]\n",
+ "box = [\n",
+ " [\n",
+ " [-54.6306579887, -25.5892766534],\n",
+ " [-54.5393341362, -25.5892766534],\n",
+ " [-54.5393341362, -25.4046299874],\n",
+ " [-54.6306579887, -25.4046299874],\n",
+ " [-54.6306579887, -25.5892766534],\n",
+ " ]\n",
+ "]\n",
"boundary = ee.Geometry.Polygon(box, None, False)\n",
"collection_name = \"COPERNICUS/S2_SR\"\n",
"scale = 10\n",
- "crs = 'EPSG:4326'\n",
+ "crs = \"EPSG:4326\"\n",
"# CRIE UMA PASTA VAZIA NO DIRETORIO RASTER E COLOQUE O CAMINHO DELA AQUI EM BAIXO\n",
"out_dir = \"raster/sentinel2_tmp\"\n",
"hdfs_dir = \"sentinel2_tmp\""
@@ -237,18 +245,24 @@
"source": [
"for i in range(2, 15, 2):\n",
" start_date = str((end_date - timedelta(days=i)))\n",
- " collection = ee.ImageCollection(collection_name) \\\n",
- " .select(['B2', 'B3', 'B4', 'B8']) \\\n",
- " .filterBounds(boundary) \\\n",
- " .filterMetadata('CLOUDY_PIXEL_PERCENTAGE', 'less_than', percent_cloud) \\\n",
+ " collection = (\n",
+ " ee.ImageCollection(collection_name)\n",
+ " .select([\"B2\", \"B3\", \"B4\", \"B8\"])\n",
+ " .filterBounds(boundary)\n",
+ " .filterMetadata(\"CLOUDY_PIXEL_PERCENTAGE\", \"less_than\", percent_cloud)\n",
" .filterDate(start_date, str(end_date))\n",
- " geemap.ee_export_image_collection(collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir)\n",
- "# NOT SURE WHAT F.. THIS BELOW DOES \n",
+ " )\n",
+ " geemap.ee_export_image_collection(\n",
+ " collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir\n",
+ " )\n",
+ " # NOT SURE WHAT F.. THIS BELOW DOES\n",
" for root, directory, files in os.walk(out_dir):\n",
- " if len(files) == 0:\n",
- " geemap.ee_export_image_collection(collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir)\n",
- " else:\n",
- " print('Existem imagens na pasta')\n"
+ " if len(files) == 0:\n",
+ " geemap.ee_export_image_collection(\n",
+ " collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir\n",
+ " )\n",
+ " else:\n",
+ " print(\"Existem imagens na pasta\")"
]
},
{
@@ -258,7 +272,7 @@
"metadata": {},
"outputs": [],
"source": [
- "hdfs = PyWebHdfsClient(host='179.106.229.159',port='50070', user_name='root')\n",
+ "hdfs = PyWebHdfsClient(host=\"179.106.229.159\", port=\"50070\", user_name=\"root\")\n",
"hdfs.delete_file_dir(hdfs_dir, recursive=True)\n",
"hdfs.make_dir(hdfs_dir)"
]
diff --git a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
index 3eb7aa31b..c1614ae75 100644
--- a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
+++ b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb
@@ -30,7 +30,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# pip install sklearn \n",
+ "# pip install sklearn\n",
"# pip install pyarrow\n",
"# pip install fsspec"
]
@@ -46,7 +46,15 @@
"from pyspark.sql import SparkSession\n",
"from pyspark import StorageLevel\n",
"import pandas as pd\n",
- "from pyspark.sql.types import StructType, StructField,StringType, LongType, IntegerType, DoubleType, ArrayType\n",
+ "from pyspark.sql.types import (\n",
+ " StructType,\n",
+ " StructField,\n",
+ " StringType,\n",
+ " LongType,\n",
+ " IntegerType,\n",
+ " DoubleType,\n",
+ " ArrayType,\n",
+ ")\n",
"from pyspark.sql.functions import regexp_replace\n",
"from sedona.register import SedonaRegistrator\n",
"from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
@@ -78,7 +86,7 @@
],
"source": [
"analise_folder = \"analise_teste_\" + str(date.today())\n",
- "hdfs = PyWebHdfsClient(host='179.106.229.159',port='50070', user_name='root')\n",
+ "hdfs = PyWebHdfsClient(host=\"179.106.229.159\", port=\"50070\", user_name=\"root\")\n",
"hdfs.delete_file_dir(analise_folder, recursive=True)"
]
},
@@ -143,26 +151,29 @@
],
"source": [
"# spark.scheduler.mode', 'FAIR'\n",
- "spark = SparkSession.\\\n",
- " builder.\\\n",
- " appName(\"Sentinel-app\").\\\n",
- " enableHiveSupport().\\\n",
- " master(\"local[*]\").\\\n",
- " master(\"spark://spark-master:7077\").\\\n",
- " config(\"spark.executor.memory\", \"15G\").\\\n",
- " config(\"spark.driver.maxResultSize\", \"135G\").\\\n",
- " config(\"spark.sql.shuffle.partitions\", \"500\").\\\n",
- " config(' spark.sql.adaptive.coalescePartitions.enabled', True).\\\n",
- " config('spark.sql.adaptive.enabled', True).\\\n",
- " config('spark.sql.adaptive.coalescePartitions.initialPartitionNum', 125).\\\n",
- " config(\"spark.sql.execution.arrow.pyspark.enabled\", True).\\\n",
- " config(\"spark.sql.execution.arrow.fallback.enabled\", True).\\\n",
- " config('spark.kryoserializer.buffer.max', 2047).\\\n",
- " config(\"spark.serializer\", KryoSerializer.getName).\\\n",
- " config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName).\\\n",
- " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\") .\\\n",
- " enableHiveSupport().\\\n",
- " getOrCreate()\n",
+ "spark = (\n",
+ " SparkSession.builder.appName(\"Sentinel-app\")\n",
+ " .enableHiveSupport()\n",
+ " .master(\"local[*]\")\n",
+ " .master(\"spark://spark-master:7077\")\n",
+ " .config(\"spark.executor.memory\", \"15G\")\n",
+ " .config(\"spark.driver.maxResultSize\", \"135G\")\n",
+ " .config(\"spark.sql.shuffle.partitions\", \"500\")\n",
+ " .config(\" spark.sql.adaptive.coalescePartitions.enabled\", True)\n",
+ " .config(\"spark.sql.adaptive.enabled\", True)\n",
+ " .config(\"spark.sql.adaptive.coalescePartitions.initialPartitionNum\", 125)\n",
+ " .config(\"spark.sql.execution.arrow.pyspark.enabled\", True)\n",
+ " .config(\"spark.sql.execution.arrow.fallback.enabled\", True)\n",
+ " .config(\"spark.kryoserializer.buffer.max\", 2047)\n",
+ " .config(\"spark.serializer\", KryoSerializer.getName)\n",
+ " .config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\",\n",
+ " )\n",
+ " .enableHiveSupport()\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"SedonaRegistrator.registerAll(spark)\n",
"sc = spark.sparkContext"
@@ -183,9 +194,9 @@
}
],
"source": [
- "# Path to directory of geotiff images \n",
+ "# Path to directory of geotiff images\n",
"DATA_DIR = \"hdfs://776faf4d6a1e:8020/sentinel2_tmp/*\"\n",
- "df = spark.read.format(\"geotiff\").option(\"dropInvalid\",True).load(DATA_DIR)"
+ "df = spark.read.format(\"geotiff\").option(\"dropInvalid\", True).load(DATA_DIR)"
]
},
{
@@ -204,8 +215,8 @@
],
"source": [
"# SUPER IMPORTANT ULTRA MEGA POWER FOR MEMORY PROBLENS SOLVE\n",
- "rdd = spark.sparkContext.parallelize((0,20))\n",
- "print(\"From local[5]\"+str(rdd.getNumPartitions()))"
+ "rdd = spark.sparkContext.parallelize((0, 20))\n",
+ "print(\"From local[5]\" + str(rdd.getNumPartitions()))"
]
},
{
@@ -295,7 +306,8 @@
}
],
"source": [
- "from pyspark.sql.functions import monotonically_increasing_id \n",
+ "from pyspark.sql.functions import monotonically_increasing_id\n",
+ "\n",
"# add ID\n",
"df_index = df.select(\"*\").withColumn(\"id\", monotonically_increasing_id())\n",
"df_index.explain()\n",
@@ -325,10 +337,15 @@
}
],
"source": [
- "# \"image.wkt as Geom\", \n",
- "df_export = df_index.selectExpr(\"id\",\"image.origin as origin\",\n",
- " \"image.height as height\", \"image.width as width\", \n",
- " \"cast(image.data as string) as data\", \"image.nBands as bands\")\n",
+ "# \"image.wkt as Geom\",\n",
+ "df_export = df_index.selectExpr(\n",
+ " \"id\",\n",
+ " \"image.origin as origin\",\n",
+ " \"image.height as height\",\n",
+ " \"image.width as width\",\n",
+ " \"cast(image.data as string) as data\",\n",
+ " \"image.nBands as bands\",\n",
+ ")\n",
"print(df_export.dtypes)\n",
"df_export.explain()\n",
"df_export.createOrReplaceTempView(\"df_export\")"
@@ -372,7 +389,7 @@
"# part_df_export.take(3)\n",
"part_df_export = df_export.take(1)\n",
"# print(part_df_export)\n",
- "pd.DataFrame(part_df_export).to_csv(\"teste.csv\", sep=',', encoding='utf-8')"
+ "pd.DataFrame(part_df_export).to_csv(\"teste.csv\", sep=\",\", encoding=\"utf-8\")"
]
},
{
@@ -417,9 +434,14 @@
}
],
"source": [
- "df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \n",
- " \"image.height as height\", \"image.width as width\", \"image.data as data\", \n",
- " \"image.nBands as bands\").cache()\n",
+ "df = df.selectExpr(\n",
+ " \"image.origin as origin\",\n",
+ " \"ST_GeomFromWkt(image.wkt) as Geom\",\n",
+ " \"image.height as height\",\n",
+ " \"image.width as width\",\n",
+ " \"image.data as data\",\n",
+ " \"image.nBands as bands\",\n",
+ ").cache()\n",
"df.show(5)\n",
"print(df.dtypes)\n",
"df.explain()"
@@ -465,15 +487,20 @@
}
],
"source": [
- "df = df.selectExpr(\"origin\", \"Geom\",\"RS_GetBand(data, 1,bands) as B2\",\"RS_GetBand(data, 2,bands) as B3\",\n",
- " \"RS_GetBand(data, 3,bands) as B4\", \n",
- " \"RS_GetBand(data, 4,bands) as B8\", \n",
- " \"RS_Array(height * width, 2.4) as constant_evi_2\",\n",
- " \"RS_Array(height * width, 2.5) as constant_evi_1\",\n",
- " \"RS_Array(height * width, 1.0) as constant_evi_3\",\n",
- " \"RS_Array(height * width, -0.5) as constant_tgi_1\",\n",
- " \"RS_Array(height * width, 120.0) as constant_tgi_2\",\n",
- " \"RS_Array(height * width, 0.001) as corrector\").cache()\n",
+ "df = df.selectExpr(\n",
+ " \"origin\",\n",
+ " \"Geom\",\n",
+ " \"RS_GetBand(data, 1,bands) as B2\",\n",
+ " \"RS_GetBand(data, 2,bands) as B3\",\n",
+ " \"RS_GetBand(data, 3,bands) as B4\",\n",
+ " \"RS_GetBand(data, 4,bands) as B8\",\n",
+ " \"RS_Array(height * width, 2.4) as constant_evi_2\",\n",
+ " \"RS_Array(height * width, 2.5) as constant_evi_1\",\n",
+ " \"RS_Array(height * width, 1.0) as constant_evi_3\",\n",
+ " \"RS_Array(height * width, -0.5) as constant_tgi_1\",\n",
+ " \"RS_Array(height * width, 120.0) as constant_tgi_2\",\n",
+ " \"RS_Array(height * width, 0.001) as corrector\",\n",
+ ").cache()\n",
"df.createOrReplaceTempView(\"allbands\")\n",
"df.show(5)"
]
@@ -517,14 +544,16 @@
}
],
"source": [
- "# Não tem data da imagem \n",
+ "# Não tem data da imagem\n",
"# Não tem parte a qual ela se refere\n",
"# Necessário adicionar\n",
"origin = df.selectExpr(\"origin\")\n",
- "split_origin = origin.select(split(col(\"origin\"),\"/\"))\n",
+ "split_origin = origin.select(split(col(\"origin\"), \"/\"))\n",
"split_origin.head()\n",
"# 20211226T134212\n",
- "split_origin = spark.sql(\"select to_timestamp(REPLACE(SPLIT(SPLIT(origin,'/')[5], '_')[1],'T',' '),'yyyyMMdd HHmmss') as image_date, SPLIT(origin,'/')[4] as feature_name, * from allbands\")\n",
+ "split_origin = spark.sql(\n",
+ " \"select to_timestamp(REPLACE(SPLIT(SPLIT(origin,'/')[5], '_')[1],'T',' '),'yyyyMMdd HHmmss') as image_date, SPLIT(origin,'/')[4] as feature_name, * from allbands\"\n",
+ ")\n",
"split_origin.show(5)"
]
},
@@ -550,33 +579,43 @@
],
"source": [
"# Fator de correcao da banda para ficar com valores entre 0 e 1\n",
- "correct_origin = split_origin.selectExpr(\"RS_MultiplyBands(B2, corrector) as bluen\",\n",
- " \"RS_MultiplyBands(B3, corrector) as greenn\", \n",
- " \"RS_MultiplyBands(B4, corrector) as redn\",\n",
- " \"RS_MultiplyBands(B8, corrector) as nirn\", \n",
- " \"*\").cache()\n",
- "correct_origin = correct_origin.selectExpr(\"RS_NormalizedDifference(nirn, redn) as gndvi\",\n",
- " \"RS_SubtractBands(nirn, redn) as sub_nirn_redn\", \n",
- " \"RS_AddBands(nirn,constant_evi_2) as add_nirn_contant_evi_2\",\n",
- " \"RS_AddBands(redn, constant_evi_3) as add_redn_contant_evi_3\", \n",
- " \"RS_DivideBands(nirn, greenn) as div_nirn_greenn\",\n",
- " \"RS_SubtractBands(greenn, redn) as sub_greenn_redn\",\n",
- " \"RS_SubtractBands(redn, greenn) as sub_redn_greenn\",\n",
- " \"RS_SubtractBands(redn, bluen) as sub_redn_bluen\",\n",
- " \"RS_AddBands(greenn, redn) as add_greenn_redn\",\n",
- " \"*\").cache()\n",
+ "correct_origin = split_origin.selectExpr(\n",
+ " \"RS_MultiplyBands(B2, corrector) as bluen\",\n",
+ " \"RS_MultiplyBands(B3, corrector) as greenn\",\n",
+ " \"RS_MultiplyBands(B4, corrector) as redn\",\n",
+ " \"RS_MultiplyBands(B8, corrector) as nirn\",\n",
+ " \"*\",\n",
+ ").cache()\n",
+ "correct_origin = correct_origin.selectExpr(\n",
+ " \"RS_NormalizedDifference(nirn, redn) as gndvi\",\n",
+ " \"RS_SubtractBands(nirn, redn) as sub_nirn_redn\",\n",
+ " \"RS_AddBands(nirn,constant_evi_2) as add_nirn_contant_evi_2\",\n",
+ " \"RS_AddBands(redn, constant_evi_3) as add_redn_contant_evi_3\",\n",
+ " \"RS_DivideBands(nirn, greenn) as div_nirn_greenn\",\n",
+ " \"RS_SubtractBands(greenn, redn) as sub_greenn_redn\",\n",
+ " \"RS_SubtractBands(redn, greenn) as sub_redn_greenn\",\n",
+ " \"RS_SubtractBands(redn, bluen) as sub_redn_bluen\",\n",
+ " \"RS_AddBands(greenn, redn) as add_greenn_redn\",\n",
+ " \"*\",\n",
+ ").cache()\n",
"\n",
- "correct_origin = correct_origin.selectExpr(\"RS_SubtractBands(add_greenn_redn, bluen) as greenn_redn_sub_bluen\",\n",
- " \"RS_AddBands(add_greenn_redn, bluen) as greenn_redn_add_bluen\",\n",
- " \"RS_SubtractBands(sub_greenn_redn, bluen) as sub_greenn_redn_bluen\",\n",
- " \"RS_SubtractBands(sub_redn_greenn, constant_tgi_2) as sub_red_gren_tgi_2\",\n",
- " \"*\").cache()\n",
- "correct_origin = correct_origin.selectExpr(\"RS_MultiplyFactor(sub_redn_bluen,120) as ms_redn_bluen_120\",\n",
- " \"*\").cache()\n",
- "correct_origin = correct_origin.selectExpr(\"RS_MultiplyFactor(sub_redn_greenn,190) as ms_redn_greenn_190\",\n",
- " \"*\").cache()\n",
- "correct_origin = correct_origin.selectExpr(\"RS_SubtractBands(ms_redn_greenn_190,ms_redn_bluen_120) as sub_msrg_190_msrb_120\",\n",
- " \"*\").cache()"
+ "correct_origin = correct_origin.selectExpr(\n",
+ " \"RS_SubtractBands(add_greenn_redn, bluen) as greenn_redn_sub_bluen\",\n",
+ " \"RS_AddBands(add_greenn_redn, bluen) as greenn_redn_add_bluen\",\n",
+ " \"RS_SubtractBands(sub_greenn_redn, bluen) as sub_greenn_redn_bluen\",\n",
+ " \"RS_SubtractBands(sub_redn_greenn, constant_tgi_2) as sub_red_gren_tgi_2\",\n",
+ " \"*\",\n",
+ ").cache()\n",
+ "correct_origin = correct_origin.selectExpr(\n",
+ " \"RS_MultiplyFactor(sub_redn_bluen,120) as ms_redn_bluen_120\", \"*\"\n",
+ ").cache()\n",
+ "correct_origin = correct_origin.selectExpr(\n",
+ " \"RS_MultiplyFactor(sub_redn_greenn,190) as ms_redn_greenn_190\", \"*\"\n",
+ ").cache()\n",
+ "correct_origin = correct_origin.selectExpr(\n",
+ " \"RS_SubtractBands(ms_redn_greenn_190,ms_redn_bluen_120) as sub_msrg_190_msrb_120\",\n",
+ " \"*\",\n",
+ ").cache()"
]
},
{
@@ -636,24 +675,29 @@
}
],
"source": [
- " # bluen = src.read(1, masked=True) / 10000\n",
- " # greenn = src.read(2, masked=True) / 10000\n",
- " # redn = src.read(3, masked=True) / 10000\n",
- " # nirn = src.read(4, masked=True) / 10000\n",
- " # evi = 2.5 * (nirn - redn) / (nirn + 2.4 * redn + 1)\n",
- " # gci = (nirn / greenn) - 1\n",
- " # gli = (2 * greenn - redn - bluen) / (2 * greenn + redn + bluen)\n",
- " # gndvi = (nirn - greenn) / (nirn + greenn)\n",
- " # tgi = (-0.5) * (190 * (redn - greenn) - 120 * (redn - bluen))\n",
- " # vari = (greenn - redn) / (greenn + redn - bluen)\n",
+ "# bluen = src.read(1, masked=True) / 10000\n",
+ "# greenn = src.read(2, masked=True) / 10000\n",
+ "# redn = src.read(3, masked=True) / 10000\n",
+ "# nirn = src.read(4, masked=True) / 10000\n",
+ "# evi = 2.5 * (nirn - redn) / (nirn + 2.4 * redn + 1)\n",
+ "# gci = (nirn / greenn) - 1\n",
+ "# gli = (2 * greenn - redn - bluen) / (2 * greenn + redn + bluen)\n",
+ "# gndvi = (nirn - greenn) / (nirn + greenn)\n",
+ "# tgi = (-0.5) * (190 * (redn - greenn) - 120 * (redn - bluen))\n",
+ "# vari = (greenn - redn) / (greenn + redn - bluen)\n",
"\n",
"\n",
- "calculated = correct_origin.selectExpr(\"RS_NormalizedDifference(nirn, redn) as gndvi\",\n",
- " \"RS_DivideBands(RS_MultiplyBands(constant_evi_1, sub_nirn_redn), RS_MultiplyBands(add_nirn_contant_evi_2, add_redn_contant_evi_3)) as evi\",\n",
- " \"RS_SubtractBands(div_nirn_greenn, constant_evi_3) as gci\",\n",
- " \"RS_DivideBands(sub_greenn_redn, greenn_redn_sub_bluen) as vari\",\n",
- " \"RS_DivideBands(RS_MultiplyFactor(sub_greenn_redn_bluen,2),RS_MultiplyFactor(greenn_redn_add_bluen, 2)) as gli\",\n",
- " \"RS_MultiplyBands(constant_tgi_1,sub_msrg_190_msrb_120) as tgi\", \"origin\", \"image_date\", \"feature_name\").cache()\n",
+ "calculated = correct_origin.selectExpr(\n",
+ " \"RS_NormalizedDifference(nirn, redn) as gndvi\",\n",
+ " \"RS_DivideBands(RS_MultiplyBands(constant_evi_1, sub_nirn_redn), RS_MultiplyBands(add_nirn_contant_evi_2, add_redn_contant_evi_3)) as evi\",\n",
+ " \"RS_SubtractBands(div_nirn_greenn, constant_evi_3) as gci\",\n",
+ " \"RS_DivideBands(sub_greenn_redn, greenn_redn_sub_bluen) as vari\",\n",
+ " \"RS_DivideBands(RS_MultiplyFactor(sub_greenn_redn_bluen,2),RS_MultiplyFactor(greenn_redn_add_bluen, 2)) as gli\",\n",
+ " \"RS_MultiplyBands(constant_tgi_1,sub_msrg_190_msrb_120) as tgi\",\n",
+ " \"origin\",\n",
+ " \"image_date\",\n",
+ " \"feature_name\",\n",
+ ").cache()\n",
"calculated.show(5)\n",
"calculated.printSchema()"
]
@@ -709,12 +753,17 @@
}
],
"source": [
- "calculated_mean = calculated.selectExpr(\"RS_Mean(gndvi) as gndvi\",\n",
- " \"RS_Mean(evi) as evi\",\n",
- " \"RS_Mean(gci) as gci\",\n",
- " \"RS_Mean(vari) as vari\",\n",
- " \"RS_Mean(gli) as gli\",\n",
- " \"RS_Mean(tgi) as tgi\", \"origin\", \"image_date\", \"feature_name\").cache()\n",
+ "calculated_mean = calculated.selectExpr(\n",
+ " \"RS_Mean(gndvi) as gndvi\",\n",
+ " \"RS_Mean(evi) as evi\",\n",
+ " \"RS_Mean(gci) as gci\",\n",
+ " \"RS_Mean(vari) as vari\",\n",
+ " \"RS_Mean(gli) as gli\",\n",
+ " \"RS_Mean(tgi) as tgi\",\n",
+ " \"origin\",\n",
+ " \"image_date\",\n",
+ " \"feature_name\",\n",
+ ").cache()\n",
"calculated_mean.show(5)\n",
"calculated_mean.printSchema()\n",
"calculated_mean.createOrReplaceTempView(\"all_mean\")"
@@ -740,7 +789,7 @@
"# part_df_export.take(3)\n",
"part_df_export = calculated_mean.limit(10).collect()\n",
"print(part_df_export)\n",
- "pd.DataFrame(part_df_export).to_csv(\"teste.csv\", sep=',', encoding='utf-8')"
+ "pd.DataFrame(part_df_export).to_csv(\"teste.csv\", sep=\",\", encoding=\"utf-8\")"
]
},
{
@@ -761,6 +810,7 @@
"# SAVE COPY TO HDFS\n",
"# dá o mesmo problema de threadshod unsuficiente que ocorre no fit\n",
"import gc\n",
+ "\n",
"collected = gc.collect()\n",
"print(\"Garbage collector: collected %d objects.\" % collected)"
]
@@ -786,7 +836,12 @@
"from pyspark.ml import Pipeline\n",
"from pyspark.ml.classification import RandomForestClassifier\n",
"from pyspark.ml.linalg import Vectors\n",
- "from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer, VectorAssembler\n",
+ "from pyspark.ml.feature import (\n",
+ " IndexToString,\n",
+ " StringIndexer,\n",
+ " VectorIndexer,\n",
+ " VectorAssembler,\n",
+ ")\n",
"from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
"from pyspark.ml.tuning import ParamGridBuilder\n",
"import numpy as np\n",
@@ -834,12 +889,14 @@
}
],
"source": [
- "vari = calculated_mean.select('vari')\n",
+ "vari = calculated_mean.select(\"vari\")\n",
"vari.printSchema()\n",
"vari.show(5)\n",
- "df_rf_assembler = calculated_mean.selectExpr(\"vari\",\"gndvi\",\"evi\",\"tgi\",\"gli\",\"cast(feature_name as long) as labels\")\n",
+ "df_rf_assembler = calculated_mean.selectExpr(\n",
+ " \"vari\", \"gndvi\", \"evi\", \"tgi\", \"gli\", \"cast(feature_name as long) as labels\"\n",
+ ")\n",
"# FORMATO NECESSARIO PARA O FIT\n",
- "feature_list = [col for col in df_rf_assembler.columns if col != 'labels']\n",
+ "feature_list = [col for col in df_rf_assembler.columns if col != \"labels\"]\n",
"assembler = VectorAssembler(inputCols=feature_list, outputCol=\"features\")\n",
"# rf = RandomForestClassifier(labelCol=\"labels\", featuresCol=\"features\")\n",
"df_rf_assembler = assembler.transform(df_rf_assembler)\n",
@@ -881,7 +938,11 @@
"from numpy import allclose\n",
"from pyspark.ml.linalg import Vectors\n",
"from pyspark.ml.feature import StringIndexer\n",
- "from pyspark.ml.classification import RandomForestClassifier, RandomForestClassificationModel\n",
+ "from pyspark.ml.classification import (\n",
+ " RandomForestClassifier,\n",
+ " RandomForestClassificationModel,\n",
+ ")\n",
+ "\n",
"# df = spark.createDataFrame([\n",
"# (1.0, Vectors.dense(1.0)),\n",
"# (0.0, Vectors.sparse(1, [], []))], [\"label\", \"features\"])\n",
diff --git a/docs/usecases/contrib/PostgresqlConnectionApacheSedona.ipynb b/docs/usecases/contrib/PostgresqlConnectionApacheSedona.ipynb
index f188bee00..55bdfd52d 100644
--- a/docs/usecases/contrib/PostgresqlConnectionApacheSedona.ipynb
+++ b/docs/usecases/contrib/PostgresqlConnectionApacheSedona.ipynb
@@ -46,11 +46,13 @@
"source": [
"from pyspark.sql import SparkSession\n",
"\n",
- "spark = SparkSession.builder.appName(\"db-connection-2\")\\\n",
- " .master(\"spark://spark-master:7077\")\\\n",
- " .config(\"spark.executor.memory\", \"10gb\")\\\n",
- " .config(\"spark.jars\", \"postgresql-42.2.24.jar\") \\\n",
- " .getOrCreate()"
+ "spark = (\n",
+ " SparkSession.builder.appName(\"db-connection-2\")\n",
+ " .master(\"spark://spark-master:7077\")\n",
+ " .config(\"spark.executor.memory\", \"10gb\")\n",
+ " .config(\"spark.jars\", \"postgresql-42.2.24.jar\")\n",
+ " .getOrCreate()\n",
+ ")"
]
},
{
@@ -62,8 +64,15 @@
},
"outputs": [],
"source": [
- "properties = {\"user\":\"\", \"password\":\"\", \"host\":\"\", \"port\":\"\", \"database\":\"\"}\n",
- "properties[\"url\"] = \"jdbc:postgresql://\"+properties[\"host\"]+\":\"+properties[\"port\"]+\"/\"+properties[\"database\"]"
+ "properties = {\"user\": \"\", \"password\": \"\", \"host\": \"\", \"port\": \"\", \"database\": \"\"}\n",
+ "properties[\"url\"] = (\n",
+ " \"jdbc:postgresql://\"\n",
+ " + properties[\"host\"]\n",
+ " + \":\"\n",
+ " + properties[\"port\"]\n",
+ " + \"/\"\n",
+ " + properties[\"database\"]\n",
+ ")"
]
},
{
@@ -75,14 +84,17 @@
},
"outputs": [],
"source": [
- "jdbcDF = spark.read.format(\"jdbc\"). \\\n",
- "options(\n",
- " url=properties[\"url\"], # jdbc:postgresql://<host>:<port>/<database>\n",
- " dbtable='clima.t_indices_prec_cpc',\n",
- " user=properties[\"user\"],\n",
- " password=properties[\"password\"], \n",
- " driver=\"org.postgresql.Driver\") \\\n",
- ".load()"
+ "jdbcDF = (\n",
+ " spark.read.format(\"jdbc\")\n",
+ " .options(\n",
+ " url=properties[\"url\"], # jdbc:postgresql://<host>:<port>/<database>\n",
+ " dbtable=\"clima.t_indices_prec_cpc\",\n",
+ " user=properties[\"user\"],\n",
+ " password=properties[\"password\"],\n",
+ " driver=\"org.postgresql.Driver\",\n",
+ " )\n",
+ " .load()\n",
+ ")"
]
},
{
@@ -189,11 +201,13 @@
"import psycopg2\n",
"\n",
"\n",
- "connection = psycopg2.connect(user=properties[\"user\"],\n",
- " password=properties[\"password\"],\n",
- " host=properties[\"host\"],\n",
- " port=properties[\"port\"],\n",
- " database=properties[\"database\"])"
+ "connection = psycopg2.connect(\n",
+ " user=properties[\"user\"],\n",
+ " password=properties[\"password\"],\n",
+ " host=properties[\"host\"],\n",
+ " port=properties[\"port\"],\n",
+ " database=properties[\"database\"],\n",
+ ")"
]
},
{
@@ -216,9 +230,10 @@
"source": [
"%%timeit\n",
"from pandas import DataFrame\n",
+ "\n",
"cursor = connection.cursor()\n",
"cursor.execute(\"SELECT * FROM clima.t_indices_prec_cpc t Where r100mm > 2000\")\n",
- "names = [ x[0] for x in cursor.description]\n",
+ "names = [x[0] for x in cursor.description]\n",
"result = cursor.fetchall()\n",
"df = DataFrame(result, columns=names)"
]
diff --git a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
index 873b49d90..986798739 100644
--- a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
+++ b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb
@@ -34,7 +34,15 @@
"from pyspark.sql import SparkSession\n",
"from pyspark import StorageLevel\n",
"import pandas as pd\n",
- "from pyspark.sql.types import StructType, StructField,StringType, LongType, IntegerType, DoubleType, ArrayType\n",
+ "from pyspark.sql.types import (\n",
+ " StructType,\n",
+ " StructField,\n",
+ " StringType,\n",
+ " LongType,\n",
+ " IntegerType,\n",
+ " DoubleType,\n",
+ " ArrayType,\n",
+ ")\n",
"from pyspark.sql.functions import regexp_replace\n",
"from sedona.register import SedonaRegistrator\n",
"from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
@@ -45,7 +53,7 @@
"from pyspark.sql.functions import udf, lit, flatten\n",
"from pywebhdfs.webhdfs import PyWebHdfsClient\n",
"from datetime import date\n",
- "from pyspark.sql.functions import monotonically_increasing_id \n",
+ "from pyspark.sql.functions import monotonically_increasing_id\n",
"import json"
]
},
@@ -110,26 +118,29 @@
],
"source": [
"# spark.scheduler.mode', 'FAIR'\n",
- "spark = SparkSession.\\\n",
- " builder.\\\n",
- " appName(\"Overpass-API\").\\\n",
- " enableHiveSupport().\\\n",
- " master(\"local[*]\").\\\n",
- " master(\"spark://spark-master:7077\").\\\n",
- " config(\"spark.executor.memory\", \"15G\").\\\n",
- " config(\"spark.driver.maxResultSize\", \"135G\").\\\n",
- " config(\"spark.sql.shuffle.partitions\", \"500\").\\\n",
- " config(' spark.sql.adaptive.coalescePartitions.enabled', True).\\\n",
- " config('spark.sql.adaptive.enabled', True).\\\n",
- " config('spark.sql.adaptive.coalescePartitions.initialPartitionNum', 125).\\\n",
- " config(\"spark.sql.execution.arrow.pyspark.enabled\", True).\\\n",
- " config(\"spark.sql.execution.arrow.fallback.enabled\", True).\\\n",
- " config('spark.kryoserializer.buffer.max', 2047).\\\n",
- " config(\"spark.serializer\", KryoSerializer.getName).\\\n",
- " config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName).\\\n",
- " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\") .\\\n",
- " enableHiveSupport().\\\n",
- " getOrCreate()\n",
+ "spark = (\n",
+ " SparkSession.builder.appName(\"Overpass-API\")\n",
+ " .enableHiveSupport()\n",
+ " .master(\"local[*]\")\n",
+ " .master(\"spark://spark-master:7077\")\n",
+ " .config(\"spark.executor.memory\", \"15G\")\n",
+ " .config(\"spark.driver.maxResultSize\", \"135G\")\n",
+ " .config(\"spark.sql.shuffle.partitions\", \"500\")\n",
+ " .config(\" spark.sql.adaptive.coalescePartitions.enabled\", True)\n",
+ " .config(\"spark.sql.adaptive.enabled\", True)\n",
+ " .config(\"spark.sql.adaptive.coalescePartitions.initialPartitionNum\", 125)\n",
+ " .config(\"spark.sql.execution.arrow.pyspark.enabled\", True)\n",
+ " .config(\"spark.sql.execution.arrow.fallback.enabled\", True)\n",
+ " .config(\"spark.kryoserializer.buffer.max\", 2047)\n",
+ " .config(\"spark.serializer\", KryoSerializer.getName)\n",
+ " .config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\",\n",
+ " )\n",
+ " .enableHiveSupport()\n",
+ " .getOrCreate()\n",
+ ")\n",
"\n",
"SedonaRegistrator.registerAll(spark)\n",
"sc = spark.sparkContext"
@@ -174,7 +185,7 @@
"out skel qt;\n",
"\"\"\"\n",
"\n",
- "# response = requests.get(overpass_url, \n",
+ "# response = requests.get(overpass_url,\n",
"# params={'data': overpass_query})\n",
"# data = response.json()\n",
"# hdfs = PyWebHdfsClient(host='179.106.229.159',port='50070', user_name='root')\n",
@@ -198,8 +209,8 @@
}
],
"source": [
- "path = \"hdfs://776faf4d6a1e:8020/\"+file_name\n",
- "df = spark.read.json(path, multiLine = \"true\")"
+ "path = \"hdfs://776faf4d6a1e:8020/\" + file_name\n",
+ "df = spark.read.json(path, multiLine=\"true\")"
]
},
{
@@ -452,32 +463,46 @@
"ids = pd.DataFrame(isolate_ids[\"id\"].iloc[0]).drop_duplicates()\n",
"print(ids[0].iloc[1])\n",
"\n",
- "formatted_df = tb\\\n",
- ".withColumn(\"id\", explode(\"elements.id\"))\n",
+ "formatted_df = tb.withColumn(\"id\", explode(\"elements.id\"))\n",
"\n",
"formatted_df.show(5)\n",
"\n",
- "formatted_df = tb\\\n",
- ".withColumn(\"new\", arrays_zip(\"elements.id\", \"elements.geometry\", \"elements.nodes\", \"elements.tags\"))\\\n",
- ".withColumn(\"new\", explode(\"new\"))\n",
+ "formatted_df = tb.withColumn(\n",
+ " \"new\",\n",
+ " arrays_zip(\"elements.id\", \"elements.geometry\", \"elements.nodes\", \"elements.tags\"),\n",
+ ").withColumn(\"new\", explode(\"new\"))\n",
"\n",
"formatted_df.show(5)\n",
"\n",
"# formatted_df.printSchema()\n",
"\n",
- "formatted_df = formatted_df.select(\"new.0\",\"new.1\",\"new.2\",\"new.3.maxspeed\",\"new.3.incline\",\"new.3.surface\", \"new.3.name\", \"total_nodes\")\n",
- "formatted_df = formatted_df.withColumnRenamed(\"0\",\"id\").withColumnRenamed(\"1\",\"geom\").withColumnRenamed(\"2\",\"nodes\").withColumnRenamed(\"3\",\"tags\")\n",
+ "formatted_df = formatted_df.select(\n",
+ " \"new.0\",\n",
+ " \"new.1\",\n",
+ " \"new.2\",\n",
+ " \"new.3.maxspeed\",\n",
+ " \"new.3.incline\",\n",
+ " \"new.3.surface\",\n",
+ " \"new.3.name\",\n",
+ " \"total_nodes\",\n",
+ ")\n",
+ "formatted_df = (\n",
+ " formatted_df.withColumnRenamed(\"0\", \"id\")\n",
+ " .withColumnRenamed(\"1\", \"geom\")\n",
+ " .withColumnRenamed(\"2\", \"nodes\")\n",
+ " .withColumnRenamed(\"3\", \"tags\")\n",
+ ")\n",
"formatted_df.createOrReplaceTempView(\"formatted_df\")\n",
"formatted_df.show(5)\n",
"# TODO atualizar daqui para baixo para considerar a linha inteira na lógica\n",
"points_tb = spark.sql(\"select geom, id from formatted_df where geom IS NOT NULL\")\n",
- "points_tb = points_tb\\\n",
- ".withColumn(\"new\", arrays_zip(\"geom.lat\", \"geom.lon\"))\\\n",
- ".withColumn(\"new\", explode(\"new\"))\n",
+ "points_tb = points_tb.withColumn(\"new\", arrays_zip(\"geom.lat\", \"geom.lon\")).withColumn(\n",
+ " \"new\", explode(\"new\")\n",
+ ")\n",
"\n",
- "points_tb = points_tb.select(\"new.0\",\"new.1\", \"id\")\n",
+ "points_tb = points_tb.select(\"new.0\", \"new.1\", \"id\")\n",
"\n",
- "points_tb = points_tb.withColumnRenamed(\"0\",\"lat\").withColumnRenamed(\"1\",\"lon\")\n",
+ "points_tb = points_tb.withColumnRenamed(\"0\", \"lat\").withColumnRenamed(\"1\", \"lon\")\n",
"points_tb.printSchema()\n",
"\n",
"points_tb.createOrReplaceTempView(\"points_tb\")\n",
@@ -496,11 +521,15 @@
"# \t\t\t\tST_Point(20, 80))\n",
"# \t\t\t\t)) As wktenv;\n",
"\n",
- "coordinates_tb = spark.sql(\"select (select collect_list(CONCAT(p1.lat,',',p1.lon)) from points_tb p1 where p1.id = p2.id group by p1.id) as coordinates, p2.id, p2.maxspeed, p2.incline, p2.surface, p2.name, p2.nodes, p2.total_nodes from formatted_df p2\")\n",
+ "coordinates_tb = spark.sql(\n",
+ " \"select (select collect_list(CONCAT(p1.lat,',',p1.lon)) from points_tb p1 where p1.id = p2.id group by p1.id) as coordinates, p2.id, p2.maxspeed, p2.incline, p2.surface, p2.name, p2.nodes, p2.total_nodes from formatted_df p2\"\n",
+ ")\n",
"coordinates_tb.createOrReplaceTempView(\"coordinates_tb\")\n",
"coordinates_tb.show(5)\n",
"\n",
- "roads_tb = spark.sql(\"SELECT ST_LineStringFromText(REPLACE(REPLACE(CAST(coordinates as string),'[',''),']',''), ',') as geom, id, maxspeed, incline, surface, name, nodes, total_nodes FROM coordinates_tb WHERE coordinates IS NOT NULL\")\n",
+ "roads_tb = spark.sql(\n",
+ " \"SELECT ST_LineStringFromText(REPLACE(REPLACE(CAST(coordinates as string),'[',''),']',''), ',') as geom, id, maxspeed, incline, surface, name, nodes, total_nodes FROM coordinates_tb WHERE coordinates IS NOT NULL\"\n",
+ ")\n",
"roads_tb.createOrReplaceTempView(\"roads_tb\")\n",
"roads_tb.show(5)"
]
@@ -520,8 +549,7 @@
}
],
"source": [
- "\n",
- "print(roads_tb.select('geom').take(1))"
+ "print(roads_tb.select(\"geom\").take(1))"
]
},
{
@@ -598,37 +626,47 @@
"# Não foi considerado que um caminha pode necessitar mais de 1 rua\n",
"\n",
"start_point = \"-25.4695946,-54.5909028\"\n",
- "end_point = \"-25.4786993,-54.57938\" \n",
+ "end_point = \"-25.4786993,-54.57938\"\n",
"\n",
- "distance_tb = spark.sql(\"select nodes, st_distance(geom, st_point(\"+end_point+\")) as distance_toend, st_distance(st_point(\"+start_point+\"), geom) as distance, st_length(geom) * 1000 as geomsize, geom, id, maxspeed, incline, surface, name , total_nodes from roads_tb\")\n",
+ "distance_tb = spark.sql(\n",
+ " \"select nodes, st_distance(geom, st_point(\"\n",
+ " + end_point\n",
+ " + \")) as distance_toend, st_distance(st_point(\"\n",
+ " + start_point\n",
+ " + \"), geom) as distance, st_length(geom) * 1000 as geomsize, geom, id, maxspeed, incline, surface, name , total_nodes from roads_tb\"\n",
+ ")\n",
"distance_tb.createOrReplaceTempView(\"distance_tb\")\n",
"distance_tb.show(5)\n",
"\n",
"# considerar distância, direcao(ex: 0 180 e etc), inclinacao(up, down, 0%), superficie(asphalt,paved, concrete), velocidade(60 80 50 40 e etc)\n",
"fill_null_tb = spark.sql(\n",
- " \"select nodes, IFNULL(maxspeed, 20) as maxspeed, IFNULL(incline, '0%') as incline, IFNULL(surface, 'soil') as surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from distance_tb\")\n",
+ " \"select nodes, IFNULL(maxspeed, 20) as maxspeed, IFNULL(incline, '0%') as incline, IFNULL(surface, 'soil') as surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from distance_tb\"\n",
+ ")\n",
"fill_null_tb.createOrReplaceTempView(\"fill_null_tb\")\n",
"fill_null_tb.show(5)\n",
"\n",
"surface_index_tb = spark.sql(\n",
- " \"select nodes, case surface when 'asphalt'\" +\n",
- " \"then 0.01 when 'concrete'\" + \n",
- " \"then 0.02 when 'paved'\" +\n",
- " \"then 0.03 when 'soil'\" +\n",
- " \"then 0.04 when 'unpaved'\" + \n",
- " \"then 0.04 when 'sett'\" +\n",
- " \"then 0.03 ELSE 0.05 end as surface_index,\"+ \n",
- " \"maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from fill_null_tb\")\n",
+ " \"select nodes, case surface when 'asphalt'\"\n",
+ " + \"then 0.01 when 'concrete'\"\n",
+ " + \"then 0.02 when 'paved'\"\n",
+ " + \"then 0.03 when 'soil'\"\n",
+ " + \"then 0.04 when 'unpaved'\"\n",
+ " + \"then 0.04 when 'sett'\"\n",
+ " + \"then 0.03 ELSE 0.05 end as surface_index,\"\n",
+ " + \"maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from fill_null_tb\"\n",
+ ")\n",
"surface_index_tb.createOrReplaceTempView(\"surface_index_tb\")\n",
"surface_index_tb.show(5)\n",
"\n",
"incline_index_tb = spark.sql(\n",
- " \"select nodes, case incline when 'top' then -0.10 when 'down' then 0.10 when '0%' then 0 end as incline_index, surface_index, maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from surface_index_tb\")\n",
+ " \"select nodes, case incline when 'top' then -0.10 when 'down' then 0.10 when '0%' then 0 end as incline_index, surface_index, maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from surface_index_tb\"\n",
+ ")\n",
"incline_index_tb.createOrReplaceTempView(\"incline_index_tb\")\n",
"incline_index_tb.show(5)\n",
- " \n",
+ "\n",
"weight_index_tb = spark.sql(\n",
- " \"select nodes, (maxspeed - (maxspeed * surface_index)) + (maxspeed +(maxspeed * incline_index)) as weight, incline_index, surface_index, maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from incline_index_tb WHERE geomsize IS NOT NULL\")\n",
+ " \"select nodes, (maxspeed - (maxspeed * surface_index)) + (maxspeed +(maxspeed * incline_index)) as weight, incline_index, surface_index, maxspeed, incline, surface, name, id, geom, geomsize, distance, distance_toend, total_nodes from incline_index_tb WHERE geomsize IS NOT NULL\"\n",
+ ")\n",
"weight_index_tb.createOrReplaceTempView(\"weight_index_tb\")\n",
"weight_index_tb.show(5)"
]
@@ -655,8 +693,7 @@
}
],
"source": [
- "teste = spark.sql(\n",
- " \"select min(weight) from weight_index_tb\")\n",
+ "teste = spark.sql(\"select min(weight) from weight_index_tb\")\n",
"teste.show(5)"
]
},
@@ -819,23 +856,30 @@
],
"source": [
"closestoend_tb = spark.sql(\n",
- " \"select w1.id, w1.distance_toend from weight_index_tb w1 group by w1.id, w1.distance_toend having (select min(w2.distance_toend) as distance_toend from weight_index_tb w2) = w1.distance_toend\")\n",
+ " \"select w1.id, w1.distance_toend from weight_index_tb w1 group by w1.id, w1.distance_toend having (select min(w2.distance_toend) as distance_toend from weight_index_tb w2) = w1.distance_toend\"\n",
+ ")\n",
"closestoend_tb.createOrReplaceTempView(\"closestoend_tb\")\n",
- "closestoend = closestoend_tb.take(1)[0]['id']\n",
+ "closestoend = closestoend_tb.take(1)[0][\"id\"]\n",
"print(closestoend)\n",
"\n",
"closestostart_tb = spark.sql(\n",
- " \"select w1.id, w1.distance from weight_index_tb w1 group by w1.id, w1.distance having (select min(w2.distance) as distance from weight_index_tb w2) = w1.distance\")\n",
+ " \"select w1.id, w1.distance from weight_index_tb w1 group by w1.id, w1.distance having (select min(w2.distance) as distance from weight_index_tb w2) = w1.distance\"\n",
+ ")\n",
"closestostart_tb.createOrReplaceTempView(\"closestostart_tb\")\n",
"closestostart_tb.show(5)\n",
"\n",
- "closestostart = closestostart_tb.take(1)[0]['id']\n",
+ "closestostart = closestostart_tb.take(1)[0][\"id\"]\n",
"\n",
"# FOLIUM EM 3857 dado em 4326 st_transform(st_union_aggr(geom),'epsg:3857','epsg:4326')\n",
"json_lines = spark.sql(\n",
- " \"select ST_AsGeoJSON(st_envelope_aggr(geom)) AS json from weight_index_tb where id in (\"+str(closestostart)+\",\"+str(closestoend)+\")\")\n",
- "json_lines_string_teste = json_lines.take(1)[0]['json']\n",
- "coordinates_teste = json.loads(json_lines_string_teste)['coordinates']\n",
+ " \"select ST_AsGeoJSON(st_envelope_aggr(geom)) AS json from weight_index_tb where id in (\"\n",
+ " + str(closestostart)\n",
+ " + \",\"\n",
+ " + str(closestoend)\n",
+ " + \")\"\n",
+ ")\n",
+ "json_lines_string_teste = json_lines.take(1)[0][\"json\"]\n",
+ "coordinates_teste = json.loads(json_lines_string_teste)[\"coordinates\"]\n",
"\n",
"\n",
"# st_boundary st_contains\n",
@@ -843,21 +887,33 @@
"# Pegar o limite entre a uniao da geom inicial e final\n",
"# select st_boundary(st_union_aggr(geom)) AS boundary from weight_index_tb where id in (\"+str(closestostart)+\",\"+str(closestoend)+\")\n",
"\n",
- "boundary_tb = spark.sql(\"select st_envelope_aggr(geom) as boundary from weight_index_tb where id in (\"+str(closestostart)+\",\"+str(closestoend)+\")\")\n",
+ "boundary_tb = spark.sql(\n",
+ " \"select st_envelope_aggr(geom) as boundary from weight_index_tb where id in (\"\n",
+ " + str(closestostart)\n",
+ " + \",\"\n",
+ " + str(closestoend)\n",
+ " + \")\"\n",
+ ")\n",
"boundary_tb.createOrReplaceTempView(\"boundary_tb\")\n",
"boundary_tb.show(5)\n",
"\n",
- "contains_tb = spark.sql(\"select st_intersects(boundary,geom) as contains, id from weight_index_tb, boundary_tb\")\n",
+ "contains_tb = spark.sql(\n",
+ " \"select st_intersects(boundary,geom) as contains, id from weight_index_tb, boundary_tb\"\n",
+ ")\n",
"contains_tb.createOrReplaceTempView(\"contains_tb\")\n",
"contains_tb.show(5)\n",
"\n",
- "possible_paths = spark.sql(\"select id, geom from weight_index_tb group by id, geom having id in (select id from contains_tb where contains = true)\")\n",
+ "possible_paths = spark.sql(\n",
+ " \"select id, geom from weight_index_tb group by id, geom having id in (select id from contains_tb where contains = true)\"\n",
+ ")\n",
"possible_paths.createOrReplaceTempView(\"possible_paths\")\n",
"possible_paths.show(5)\n",
"\n",
- "paths_collection = spark.sql(\"select ST_AsGeoJSON(st_union_aggr(geom)) AS json from possible_paths\")\n",
- "json_lines_string = paths_collection.take(1)[0]['json']\n",
- "coordinates = json.loads(json_lines_string)['coordinates']"
+ "paths_collection = spark.sql(\n",
+ " \"select ST_AsGeoJSON(st_union_aggr(geom)) AS json from possible_paths\"\n",
+ ")\n",
+ "json_lines_string = paths_collection.take(1)[0][\"json\"]\n",
+ "coordinates = json.loads(json_lines_string)[\"coordinates\"]"
]
},
{
@@ -1010,47 +1066,60 @@
"source": [
"path = [closestostart]\n",
"visited = [closestostart]\n",
- "current_nodes = spark.sql(\"select geom from weight_index_tb where id = \"+str(closestostart))\n",
+ "current_nodes = spark.sql(\n",
+ " \"select geom from weight_index_tb where id = \" + str(closestostart)\n",
+ ")\n",
"row = current_nodes.rdd.collect()[0][\"geom\"]\n",
"id_current = closestostart\n",
"\n",
+ "\n",
"def choose_path(row, path, id_current, visited, copy_row):\n",
"\n",
- " visited_frm = str(visited).replace(\"[\",\"(\").replace(\"]\",\")\")\n",
- " \n",
- " touches_tb = spark.sql(\"select st_touches(st_geomfromwkt('\"+str(row)+\"'),geom) as touches, * from weight_index_tb where geom IS NOT NULL and distance_toend IS NOT NULL\")\n",
+ " visited_frm = str(visited).replace(\"[\", \"(\").replace(\"]\", \")\")\n",
+ "\n",
+ " touches_tb = spark.sql(\n",
+ " \"select st_touches(st_geomfromwkt('\"\n",
+ " + str(row)\n",
+ " + \"'),geom) as touches, * from weight_index_tb where geom IS NOT NULL and distance_toend IS NOT NULL\"\n",
+ " )\n",
" touches_tb.createOrReplaceTempView(\"touches_tb\")\n",
- "# st_distance(st_geomfromwkt('\"+str(row)+\"'),geom)\n",
- " fim_distance = spark.sql(\"select distance_toend from touches_tb where id = \"+ str(id_current))\n",
+ " # st_distance(st_geomfromwkt('\"+str(row)+\"'),geom)\n",
+ " fim_distance = spark.sql(\n",
+ " \"select distance_toend from touches_tb where id = \" + str(id_current)\n",
+ " )\n",
" fim_distance.show(5)\n",
" fim_distance_value = fim_distance.rdd.collect()[0][\"distance_toend\"]\n",
- " \n",
- " \n",
- " current_distance = spark.sql(\"select distance from touches_tb where id = \" + str(id_current))\n",
+ "\n",
+ " current_distance = spark.sql(\n",
+ " \"select distance from touches_tb where id = \" + str(id_current)\n",
+ " )\n",
" current_distance_value = current_distance.rdd.collect()[0][\"distance\"]\n",
- " \n",
- "# st_distance(st_geomfromwkt('\"+str(row)+\"'),geom) = \n",
"\n",
- " sql = \"select geom, id, weight from touches_tb where \" \\\n",
- " +\"touches = true\" \\\n",
- " +\" and \" \\\n",
- " +\"distance_toend < \" \\\n",
- " +str(fim_distance_value) \\\n",
- " +\" and \" \\\n",
- " +\"distance > \" \\\n",
- " +str(current_distance_value) \\\n",
- " +\" and \" \\\n",
- " +\"id NOT IN \" \\\n",
- " +visited_frm \\\n",
- " \n",
+ " # st_distance(st_geomfromwkt('\"+str(row)+\"'),geom) =\n",
+ "\n",
+ " sql = (\n",
+ " \"select geom, id, weight from touches_tb where \"\n",
+ " + \"touches = true\"\n",
+ " + \" and \"\n",
+ " + \"distance_toend < \"\n",
+ " + str(fim_distance_value)\n",
+ " + \" and \"\n",
+ " + \"distance > \"\n",
+ " + str(current_distance_value)\n",
+ " + \" and \"\n",
+ " + \"id NOT IN \"\n",
+ " + visited_frm\n",
+ " )\n",
" print(sql)\n",
" current_nodes = spark.sql(sql)\n",
" current_nodes.createOrReplaceTempView(\"current_nodes\")\n",
" current_nodes.show(5)\n",
- " \n",
- " current_node = spark.sql(\"select id, geom, max(weight) from current_nodes group by id, geom, weight having max(weight) = weight \")\n",
+ "\n",
+ " current_node = spark.sql(\n",
+ " \"select id, geom, max(weight) from current_nodes group by id, geom, weight having max(weight) = weight \"\n",
+ " )\n",
" current_node.show(5)\n",
- " \n",
+ "\n",
" if len(current_nodes.rdd.collect()) == 0:\n",
" return path\n",
" else:\n",
@@ -1059,15 +1128,19 @@
" path.append(id_current)\n",
" visited.append(id_current)\n",
" return choose_path(row, path, id_current, visited, copy_row)\n",
- " \n",
+ "\n",
+ "\n",
"path_ids = choose_path(row, path, id_current, visited, row)\n",
"path_ids.append(closestoend)\n",
- "path_ids_frm = str(path_ids).replace(\"[\",\"(\").replace(\"]\",\")\")\n",
+ "path_ids_frm = str(path_ids).replace(\"[\", \"(\").replace(\"]\", \")\")\n",
"print(path_ids_frm)\n",
"\n",
- "short_path = spark.sql(\"select ST_AsGeoJSON(st_union_aggr(geom)) AS json from weight_index_tb where id in \"+path_ids_frm)\n",
- "short_path_string = short_path.take(1)[0]['json']\n",
- "short_path_coordinates = json.loads(short_path_string)['coordinates']"
+ "short_path = spark.sql(\n",
+ " \"select ST_AsGeoJSON(st_union_aggr(geom)) AS json from weight_index_tb where id in \"\n",
+ " + path_ids_frm\n",
+ ")\n",
+ "short_path_string = short_path.take(1)[0][\"json\"]\n",
+ "short_path_coordinates = json.loads(short_path_string)[\"coordinates\"]"
]
},
{
@@ -1095,13 +1168,21 @@
"\n",
"import folium\n",
"\n",
- "start_point_arr = [-25.4695946,-54.5909028]\n",
- "end_point_arr = [-25.4786993,-54.57938] \n",
+ "start_point_arr = [-25.4695946, -54.5909028]\n",
+ "end_point_arr = [-25.4786993, -54.57938]\n",
"tooltip = \"Click me!\"\n",
"# 3857\n",
- "m = folium.Map(location=[-25.5172662,-54.6170038], zoom_start=12, tiles='OpenStreetMap', crs='EPSG3857' )\n",
+ "m = folium.Map(\n",
+ " location=[-25.5172662, -54.6170038],\n",
+ " zoom_start=12,\n",
+ " tiles=\"OpenStreetMap\",\n",
+ " crs=\"EPSG3857\",\n",
+ ")\n",
"folium.Marker(\n",
- " start_point_arr, popup=\"<i>Inicio</i>\", tooltip=tooltip, icon=folium.Icon(color=\"green\")\n",
+ " start_point_arr,\n",
+ " popup=\"<i>Inicio</i>\",\n",
+ " tooltip=tooltip,\n",
+ " icon=folium.Icon(color=\"green\"),\n",
").add_to(m)\n",
"folium.Marker(\n",
" end_point_arr, popup=\"<b>Fim</b>\", tooltip=tooltip, icon=folium.Icon(color=\"red\")\n",
diff --git a/docs/usecases/contrib/foot-traffic.ipynb b/docs/usecases/contrib/foot-traffic.ipynb
index 0367463c2..200da0dc7 100644
--- a/docs/usecases/contrib/foot-traffic.ipynb
+++ b/docs/usecases/contrib/foot-traffic.ipynb
@@ -114,16 +114,18 @@
"from sedona.register import SedonaRegistrator\n",
"from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n",
"\n",
- "spark = SparkSession. \\\n",
- " builder. \\\n",
- " appName('sigspatial2021'). \\\n",
- " master(\"spark://data-ocean-lab-1:7077\").\\\n",
- " config(\"spark.serializer\", KryoSerializer.getName).\\\n",
- " config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName) .\\\n",
- " config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.2.0-incubating,'\n",
- " 'org.datasyslab:geotools-wrapper:1.1.0-25.2'). \\\n",
- " getOrCreate()"
+ "spark = (\n",
+ " SparkSession.builder.appName(\"sigspatial2021\")\n",
+ " .master(\"spark://data-ocean-lab-1:7077\")\n",
+ " .config(\"spark.serializer\", KryoSerializer.getName)\n",
+ " .config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)\n",
+ " .config(\n",
+ " \"spark.jars.packages\",\n",
+ " \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.2.0-incubating,\"\n",
+ " \"org.datasyslab:geotools-wrapper:1.1.0-25.2\",\n",
+ " )\n",
+ " .getOrCreate()\n",
+ ")"
]
},
{
@@ -183,13 +185,18 @@
}
],
"source": [
- "sample_csv_path = 'file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/seattle_coffee_monthly_patterns/'\n",
+ "sample_csv_path = \"file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/seattle_coffee_monthly_patterns/\"\n",
"sample = (\n",
- " spark.read.option(\"header\", \"true\").option(\"escape\", \"\\\"\").csv(sample_csv_path)\n",
- " .withColumn('date_range_start', f.to_date(f.col('date_range_start')))\n",
- " .withColumn('date_range_end', f.to_date(f.col('date_range_end')))\n",
- " .withColumn('visitor_home_cbgs', f.from_json('visitor_home_cbgs', schema = MapType(StringType(), IntegerType())))\n",
- " .withColumn(\"distance_from_home\", f.col(\"distance_from_home\"))\n",
+ " spark.read.option(\"header\", \"true\")\n",
+ " .option(\"escape\", '\"')\n",
+ " .csv(sample_csv_path)\n",
+ " .withColumn(\"date_range_start\", f.to_date(f.col(\"date_range_start\")))\n",
+ " .withColumn(\"date_range_end\", f.to_date(f.col(\"date_range_end\")))\n",
+ " .withColumn(\n",
+ " \"visitor_home_cbgs\",\n",
+ " f.from_json(\"visitor_home_cbgs\", schema=MapType(StringType(), IntegerType())),\n",
+ " )\n",
+ " .withColumn(\"distance_from_home\", f.col(\"distance_from_home\"))\n",
")"
]
},
@@ -561,16 +568,26 @@
},
"outputs": [],
"source": [
- "w = Window().partitionBy('placekey').orderBy(f.col('date_range_start').desc())\n",
+ "w = Window().partitionBy(\"placekey\").orderBy(f.col(\"date_range_start\").desc())\n",
"\n",
"cafes_latest = (\n",
- " sample\n",
- " # as our data improves, addresses or geocodes for a given location may change over time\n",
- " # use a window function to keep only the most recent appearance of the given cafe\n",
- " .withColumn('row_num', f.row_number().over(w))\n",
- " .filter(f.col('row_num') == 1)\n",
- " # select the columns we need for mapping\n",
- " .select('placekey', 'location_name', 'brands', 'street_address', 'city', 'region', 'postal_code', 'latitude', 'longitude', 'open_hours')\n",
+ " sample\n",
+ " # as our data improves, addresses or geocodes for a given location may change over time\n",
+ " # use a window function to keep only the most recent appearance of the given cafe\n",
+ " .withColumn(\"row_num\", f.row_number().over(w)).filter(f.col(\"row_num\") == 1)\n",
+ " # select the columns we need for mapping\n",
+ " .select(\n",
+ " \"placekey\",\n",
+ " \"location_name\",\n",
+ " \"brands\",\n",
+ " \"street_address\",\n",
+ " \"city\",\n",
+ " \"region\",\n",
+ " \"postal_code\",\n",
+ " \"latitude\",\n",
+ " \"longitude\",\n",
+ " \"open_hours\",\n",
+ " )\n",
")"
]
},
@@ -597,7 +614,11 @@
"source": [
"# create a geopandas geodataframe\n",
"cafes_gdf = cafes_latest.toPandas()\n",
- "cafes_gdf = gpd.GeoDataFrame(cafes_gdf, geometry = gpd.points_from_xy(cafes_gdf['longitude'], cafes_gdf['latitude']), crs = 'EPSG:4326')"
+ "cafes_gdf = gpd.GeoDataFrame(\n",
+ " cafes_gdf,\n",
+ " geometry=gpd.points_from_xy(cafes_gdf[\"longitude\"], cafes_gdf[\"latitude\"]),\n",
+ " crs=\"EPSG:4326\",\n",
+ ")"
]
},
{
@@ -614,37 +635,34 @@
"outputs": [],
"source": [
"def map_cafes(gdf):\n",
- " \n",
- " # map bounds\n",
- " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
- " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
- " folium_bounds = [sw, ne]\n",
- " \n",
- " # map\n",
- " x = gdf.centroid.x[0]\n",
- " y = gdf.centroid.y[0]\n",
- " \n",
- " map_ = folium.Map(\n",
- " location = [y, x],\n",
- " tiles = \"OpenStreetMap\"\n",
- " )\n",
- " \n",
- " for i, point in gdf.iterrows():\n",
- " \n",
- " tooltip = f\"placekey: {point['placekey']}<br>location_name: {point['location_name']}<br>brands: {point['brands']}<br>street_address: {point['street_address']}<br>city: {point['city']}<br>region: {point['region']}<br>postal_code: {point['postal_code']}<br>open_hours: {point['open_hours']}\"\n",
- " \n",
- " folium.Circle(\n",
- " [point['geometry'].y, point['geometry'].x],\n",
- " radius = 40,\n",
- " fill_color = 'blue',\n",
- " color = 'blue',\n",
- " fill_opacity = 1,\n",
- " tooltip = tooltip\n",
- " ).add_to(map_)\n",
"\n",
- " map_.fit_bounds(folium_bounds) \n",
- " \n",
- " return map_"
+ " # map bounds\n",
+ " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
+ " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
+ " folium_bounds = [sw, ne]\n",
+ "\n",
+ " # map\n",
+ " x = gdf.centroid.x[0]\n",
+ " y = gdf.centroid.y[0]\n",
+ "\n",
+ " map_ = folium.Map(location=[y, x], tiles=\"OpenStreetMap\")\n",
+ "\n",
+ " for i, point in gdf.iterrows():\n",
+ "\n",
+ " tooltip = f\"placekey: {point['placekey']}<br>location_name: {point['location_name']}<br>brands: {point['brands']}<br>street_address: {point['street_address']}<br>city: {point['city']}<br>region: {point['region']}<br>postal_code: {point['postal_code']}<br>open_hours: {point['open_hours']}\"\n",
+ "\n",
+ " folium.Circle(\n",
+ " [point[\"geometry\"].y, point[\"geometry\"].x],\n",
+ " radius=40,\n",
+ " fill_color=\"blue\",\n",
+ " color=\"blue\",\n",
+ " fill_opacity=1,\n",
+ " tooltip=tooltip,\n",
+ " ).add_to(map_)\n",
+ "\n",
+ " map_.fit_bounds(folium_bounds)\n",
+ "\n",
+ " return map_"
]
},
{
@@ -791,21 +809,22 @@
}
],
"source": [
- "# the `distance_from_home` column tells us the median distance (as the crow flies), in meters, between the coffee shop and the visitors' homes \n",
+ "# the `distance_from_home` column tells us the median distance (as the crow flies), in meters, between the coffee shop and the visitors' homes\n",
"# which coffee shop's visitors had the highest average median distance traveled since Jan 2018?\n",
"\n",
"# outlier values in this column distort the histogram\n",
"# these outliers are likely due to a combination of (1) coffee shops in downtown areas that receive high numbers of out-of-town visitors and (2) quirks in the underlying GPS data\n",
"furthest_traveled = (\n",
- " sample\n",
- " .groupBy('placekey', 'location_name', 'street_address')\n",
- " .agg(f.mean('distance_from_home').alias('avg_median_dist_from_home'))\n",
- " .orderBy('avg_median_dist_from_home', ascending = False)\n",
+ " sample.groupBy(\"placekey\", \"location_name\", \"street_address\")\n",
+ " .agg(f.mean(\"distance_from_home\").alias(\"avg_median_dist_from_home\"))\n",
+ " .orderBy(\"avg_median_dist_from_home\", ascending=False)\n",
")\n",
"\n",
"display(furthest_traveled)\n",
- "furthest_traveled.filter(f.col(\"avg_median_dist_from_home\").isNotNull()).withColumn(\"avg_median_dist_from_home\", f.col(\"avg_median_dist_from_home\")/1000.0).show(truncate=False)\n",
- "furthest_traveled = furthest_traveled.drop('street_address')"
+ "furthest_traveled.filter(f.col(\"avg_median_dist_from_home\").isNotNull()).withColumn(\n",
+ " \"avg_median_dist_from_home\", f.col(\"avg_median_dist_from_home\") / 1000.0\n",
+ ").show(truncate=False)\n",
+ "furthest_traveled = furthest_traveled.drop(\"street_address\")"
]
},
{
@@ -854,8 +873,13 @@
],
"source": [
"# most coffee shops' visitors' homes are <10km away\n",
- "display(furthest_traveled.filter(f.col('avg_median_dist_from_home') < 10000))\n",
- "print(furthest_traveled.filter(f.col('avg_median_dist_from_home') < 10000).count(), \" coffee shops have vistors' home <10 km away, out of \", furthest_traveled.count(), \"coffee shops.\")"
+ "display(furthest_traveled.filter(f.col(\"avg_median_dist_from_home\") < 10000))\n",
+ "print(\n",
+ " furthest_traveled.filter(f.col(\"avg_median_dist_from_home\") < 10000).count(),\n",
+ " \" coffee shops have vistors' home <10 km away, out of \",\n",
+ " furthest_traveled.count(),\n",
+ " \"coffee shops.\",\n",
+ ")"
]
},
{
@@ -905,10 +929,12 @@
],
"source": [
"# load the census block groups for Washington State\n",
- "# filter it down to our three counties of interest in Seattle \n",
+ "# filter it down to our three counties of interest in Seattle\n",
"WA_cbgs = (\n",
- " spark.read.option('header', 'true').option('escape', \"\\\"\").csv('file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/wa_cbg.csv')\n",
- " .filter(f.col('GEOID').rlike('^(53033|53053|53061)'))\n",
+ " spark.read.option(\"header\", \"true\")\n",
+ " .option(\"escape\", '\"')\n",
+ " .csv(\"file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/wa_cbg.csv\")\n",
+ " .filter(f.col(\"GEOID\").rlike(\"^(53033|53053|53061)\"))\n",
")\n",
"WA_cbgs.head()"
]
@@ -928,14 +954,18 @@
"source": [
"# transform the geometry column into a Geometry-type\n",
"WA_cbgs = (\n",
- " WA_cbgs\n",
- " .withColumn('cbg_geometry', f.expr(\"ST_GeomFromWkt(geometry)\"))\n",
- " # we'll just use the CBG centroid\n",
- " .withColumn('cbg_geometry', f.expr(\"ST_Centroid(cbg_geometry)\"))\n",
- " # since we'll be doing a distance calculation, let's also use a projected CRS - epsg:3857\n",
- " .withColumn('cbg_geometry', f.expr(\"ST_Transform(ST_FlipCoordinates(cbg_geometry), 'epsg:4326','epsg:3857', false)\")) # ST_FlipCoordinates() necessary due to this bug: https://issues.apache.org/jira/browse/SEDONA-39\n",
- " .withColumnRenamed('GEOID', 'cbg')\n",
- " .withColumnRenamed('geometry', 'cbg_polygon_geometry')\n",
+ " WA_cbgs.withColumn(\"cbg_geometry\", f.expr(\"ST_GeomFromWkt(geometry)\"))\n",
+ " # we'll just use the CBG centroid\n",
+ " .withColumn(\"cbg_geometry\", f.expr(\"ST_Centroid(cbg_geometry)\"))\n",
+ " # since we'll be doing a distance calculation, let's also use a projected CRS - epsg:3857\n",
+ " .withColumn(\n",
+ " \"cbg_geometry\",\n",
+ " f.expr(\n",
+ " \"ST_Transform(ST_FlipCoordinates(cbg_geometry), 'epsg:4326','epsg:3857', false)\"\n",
+ " ),\n",
+ " ) # ST_FlipCoordinates() necessary due to this bug: https://issues.apache.org/jira/browse/SEDONA-39\n",
+ " .withColumnRenamed(\"GEOID\", \"cbg\")\n",
+ " .withColumnRenamed(\"geometry\", \"cbg_polygon_geometry\")\n",
")"
]
},
@@ -1052,29 +1082,33 @@
"source": [
"# Next let's prep our sample data\n",
"sample_seattle_visitors = (\n",
- " sample\n",
- " .select('placekey', f.explode('visitor_home_cbgs'))\n",
- " .withColumnRenamed('key', 'cbg')\n",
- " .withColumnRenamed('value', 'visitors')\n",
- "# # filter out CBGs with low visitor counts\n",
- " .filter(f.col('visitors') > 4)\n",
- " # filter down to only the visitors from Seattle CBGs\n",
- " .filter(f.col('cbg').rlike('^(53033|53053|53061)'))\n",
- " # aggregate up all the visitors over time from each CBG to each Cafe\n",
- " .groupBy('placekey', 'cbg')\n",
- " .agg(\n",
- " f.sum('visitors').alias('visitors')\n",
- " )\n",
- " # join back with most up-to-date POI information\n",
- " .join(\n",
- " cafes_latest.select('placekey', 'latitude', 'longitude'),\n",
- " 'placekey'\n",
- " )\n",
- " # transform geometry column\n",
- " .withColumn('cafe_geometry', f.expr(\"ST_Point(CAST(longitude AS Decimal(24, 20)), CAST(latitude AS Decimal(24, 20)))\"))\n",
- " .withColumn('cafe_geometry', f.expr(\"ST_Transform(ST_FlipCoordinates(cafe_geometry), 'epsg:4326','epsg:3857', false)\"))\n",
- " # join with CBG geometries\n",
- " .join(WA_cbgs, 'cbg')\n",
+ " sample.select(\"placekey\", f.explode(\"visitor_home_cbgs\"))\n",
+ " .withColumnRenamed(\"key\", \"cbg\")\n",
+ " .withColumnRenamed(\"value\", \"visitors\")\n",
+ " # # filter out CBGs with low visitor counts\n",
+ " .filter(f.col(\"visitors\") > 4)\n",
+ " # filter down to only the visitors from Seattle CBGs\n",
+ " .filter(f.col(\"cbg\").rlike(\"^(53033|53053|53061)\"))\n",
+ " # aggregate up all the visitors over time from each CBG to each Cafe\n",
+ " .groupBy(\"placekey\", \"cbg\")\n",
+ " .agg(f.sum(\"visitors\").alias(\"visitors\"))\n",
+ " # join back with most up-to-date POI information\n",
+ " .join(cafes_latest.select(\"placekey\", \"latitude\", \"longitude\"), \"placekey\")\n",
+ " # transform geometry column\n",
+ " .withColumn(\n",
+ " \"cafe_geometry\",\n",
+ " f.expr(\n",
+ " \"ST_Point(CAST(longitude AS Decimal(24, 20)), CAST(latitude AS Decimal(24, 20)))\"\n",
+ " ),\n",
+ " )\n",
+ " .withColumn(\n",
+ " \"cafe_geometry\",\n",
+ " f.expr(\n",
+ " \"ST_Transform(ST_FlipCoordinates(cafe_geometry), 'epsg:4326','epsg:3857', false)\"\n",
+ " ),\n",
+ " )\n",
+ " # join with CBG geometries\n",
+ " .join(WA_cbgs, \"cbg\")\n",
")"
]
},
@@ -1241,11 +1275,13 @@
"outputs": [],
"source": [
"distance_traveled_SEA = (\n",
- " sample_seattle_visitors\n",
- " # calculate the distance from home in meters\n",
- " .withColumn('distance_from_home', f.expr(\"ST_Distance(cafe_geometry, cbg_geometry)\"))\n",
+ " sample_seattle_visitors\n",
+ " # calculate the distance from home in meters\n",
+ " .withColumn(\n",
+ " \"distance_from_home\", f.expr(\"ST_Distance(cafe_geometry, cbg_geometry)\")\n",
+ " )\n",
")\n",
- "distance_traveled_SEA.createOrReplaceTempView('distance_traveled_SEA')"
+ "distance_traveled_SEA.createOrReplaceTempView(\"distance_traveled_SEA\")"
]
},
{
@@ -1261,7 +1297,7 @@
},
"outputs": [],
"source": [
- "q = '''\n",
+ "q = \"\"\"\n",
"SELECT *\n",
"FROM (\n",
" SELECT DISTINCT \n",
@@ -1273,7 +1309,7 @@
" FROM distance_traveled_SEA\n",
")\n",
"WHERE pos > 0\n",
- "'''\n",
+ "\"\"\"\n",
"\n",
"weighted_median_tmp = spark.sql(q)"
]
@@ -1291,8 +1327,8 @@
},
"outputs": [],
"source": [
- "grp_window = Window.partitionBy('placekey')\n",
- "median_percentile = f.expr('percentile_approx(distance_from_home, 0.5)')"
+ "grp_window = Window.partitionBy(\"placekey\")\n",
+ "median_percentile = f.expr(\"percentile_approx(distance_from_home, 0.5)\")"
]
},
{
@@ -1424,10 +1460,8 @@
},
"outputs": [],
"source": [
- "median_dist_traveled_SEA = (\n",
- " weighted_median_tmp\n",
- " .groupBy('placekey')\n",
- " .agg(median_percentile.alias('median_dist_traveled_SEA'))\n",
+ "median_dist_traveled_SEA = weighted_median_tmp.groupBy(\"placekey\").agg(\n",
+ " median_percentile.alias(\"median_dist_traveled_SEA\")\n",
")"
]
},
@@ -1520,7 +1554,9 @@
}
],
"source": [
- "median_dist_traveled_SEA.filter(f.col('median_dist_traveled_SEA') < 25000).limit(10).toPandas().head()\n"
+ "median_dist_traveled_SEA.filter(f.col(\"median_dist_traveled_SEA\") < 25000).limit(\n",
+ " 10\n",
+ ").toPandas().head()"
]
},
{
@@ -1536,13 +1572,9 @@
},
"outputs": [],
"source": [
- "total_visits = (\n",
- " sample\n",
- " .groupBy('placekey')\n",
- " .agg(\n",
- " f.sum('raw_visit_counts').alias('total_visits'),\n",
- " f.sum('raw_visitor_counts').alias('total_visitors')\n",
- " )\n",
+ "total_visits = sample.groupBy(\"placekey\").agg(\n",
+ " f.sum(\"raw_visit_counts\").alias(\"total_visits\"),\n",
+ " f.sum(\"raw_visitor_counts\").alias(\"total_visitors\"),\n",
")"
]
},
@@ -1560,14 +1592,16 @@
"outputs": [],
"source": [
"distance_traveled_final = (\n",
- " furthest_traveled\n",
- " .join(median_dist_traveled_SEA, 'placekey')\n",
- " .join(cafes_latest, ['placekey', 'location_name'])\n",
- " .withColumn('distance_traveled_diff', f.col('avg_median_dist_from_home') - f.col('median_dist_traveled_SEA'))\n",
- " # keep only the cafes with a meaningful sample - at least 1000 visits since 2018\n",
- " .join(total_visits, 'placekey')\n",
- " .filter(f.col('total_visits') > 1000)\n",
- " .orderBy('distance_traveled_diff', ascending = False)\n",
+ " furthest_traveled.join(median_dist_traveled_SEA, \"placekey\")\n",
+ " .join(cafes_latest, [\"placekey\", \"location_name\"])\n",
+ " .withColumn(\n",
+ " \"distance_traveled_diff\",\n",
+ " f.col(\"avg_median_dist_from_home\") - f.col(\"median_dist_traveled_SEA\"),\n",
+ " )\n",
+ " # keep only the cafes with a meaningful sample - at least 1000 visits since 2018\n",
+ " .join(total_visits, \"placekey\")\n",
+ " .filter(f.col(\"total_visits\") > 1000)\n",
+ " .orderBy(\"distance_traveled_diff\", ascending=False)\n",
")"
]
},
@@ -1800,7 +1834,7 @@
}
],
"source": [
- "distance_traveled_final.select('placekey').distinct().count()"
+ "distance_traveled_final.select(\"placekey\").distinct().count()"
]
},
{
@@ -1824,14 +1858,26 @@
}
],
"source": [
- "most_tourists = distance_traveled_final.limit(500).withColumn('visitor_type', f.lit('tourist'))\n",
- "most_locals = distance_traveled_final.orderBy('distance_traveled_diff').limit(500).withColumn('visitor_type', f.lit('local'))\n",
+ "most_tourists = distance_traveled_final.limit(500).withColumn(\n",
+ " \"visitor_type\", f.lit(\"tourist\")\n",
+ ")\n",
+ "most_locals = (\n",
+ " distance_traveled_final.orderBy(\"distance_traveled_diff\")\n",
+ " .limit(500)\n",
+ " .withColumn(\"visitor_type\", f.lit(\"local\"))\n",
+ ")\n",
"\n",
"visitor_type = most_tourists.unionByName(most_locals)\n",
"\n",
"# create a geopandas geodataframe\n",
"visitor_type_gdf = visitor_type.toPandas()\n",
- "visitor_type_gdf = gpd.GeoDataFrame(visitor_type_gdf, geometry = gpd.points_from_xy(visitor_type_gdf['longitude'], visitor_type_gdf['latitude']), crs = 'EPSG:4326')"
+ "visitor_type_gdf = gpd.GeoDataFrame(\n",
+ " visitor_type_gdf,\n",
+ " geometry=gpd.points_from_xy(\n",
+ " visitor_type_gdf[\"longitude\"], visitor_type_gdf[\"latitude\"]\n",
+ " ),\n",
+ " crs=\"EPSG:4326\",\n",
+ ")"
]
},
{
@@ -1848,37 +1894,34 @@
"outputs": [],
"source": [
"def map_cafe_visitor_type(gdf):\n",
- " \n",
- " # map bounds\n",
- " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
- " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
- " folium_bounds = [sw, ne]\n",
- " \n",
- " # map\n",
- " x = gdf.centroid.x[0]\n",
- " y = gdf.centroid.y[0]\n",
- " \n",
- " map_ = folium.Map(\n",
- " location = [y, x],\n",
- " tiles = \"OpenStreetMap\"\n",
- " )\n",
- " \n",
- " for i, point in gdf.iterrows():\n",
- " \n",
- " tooltip = f\"placekey: {point['placekey']}<br>location_name: {point['location_name']}<br>brands: {point['brands']}<br>street_address: {point['street_address']}<br>city: {point['city']}<br>region: {point['region']}<br>postal_code: {point['postal_code']}<br>visitor_type: {point['visitor_type']}<br>avg_median_dist_from_home: {point['avg_median_dist_from_home']}\"\n",
- " \n",
- " folium.Circle(\n",
- " [point['geometry'].y, point['geometry'].x],\n",
- " radius = 40,\n",
- " fill_color = 'blue' if point['visitor_type'] == 'tourist' else 'red',\n",
- " color = 'blue' if point['visitor_type'] == 'tourist' else 'red',\n",
- " fill_opacity = 1,\n",
- " tooltip = tooltip\n",
- " ).add_to(map_)\n",
"\n",
- " map_.fit_bounds(folium_bounds) \n",
- " \n",
- " return map_"
+ " # map bounds\n",
+ " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
+ " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
+ " folium_bounds = [sw, ne]\n",
+ "\n",
+ " # map\n",
+ " x = gdf.centroid.x[0]\n",
+ " y = gdf.centroid.y[0]\n",
+ "\n",
+ " map_ = folium.Map(location=[y, x], tiles=\"OpenStreetMap\")\n",
+ "\n",
+ " for i, point in gdf.iterrows():\n",
+ "\n",
+ " tooltip = f\"placekey: {point['placekey']}<br>location_name: {point['location_name']}<br>brands: {point['brands']}<br>street_address: {point['street_address']}<br>city: {point['city']}<br>region: {point['region']}<br>postal_code: {point['postal_code']}<br>visitor_type: {point['visitor_type']}<br>avg_median_dist_from_home: {point['avg_median_dist_from_home']}\"\n",
+ "\n",
+ " folium.Circle(\n",
+ " [point[\"geometry\"].y, point[\"geometry\"].x],\n",
+ " radius=40,\n",
+ " fill_color=\"blue\" if point[\"visitor_type\"] == \"tourist\" else \"red\",\n",
+ " color=\"blue\" if point[\"visitor_type\"] == \"tourist\" else \"red\",\n",
+ " fill_opacity=1,\n",
+ " tooltip=tooltip,\n",
+ " ).add_to(map_)\n",
+ "\n",
+ " map_.fit_bounds(folium_bounds)\n",
+ "\n",
+ " return map_"
]
},
{
@@ -2029,11 +2072,15 @@
],
"source": [
"WA_neighbs = (\n",
- " spark.read.option('header', 'true').option('escape', \"\\\"\").csv('file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/seattle_neighborhoods.csv')\n",
- " # transform the geometry column into a Geometry-type\n",
- " .withColumn('geometry', f.expr(\"ST_GeomFromWkt(geometry)\"))\n",
+ " spark.read.option(\"header\", \"true\")\n",
+ " .option(\"escape\", '\"')\n",
+ " .csv(\n",
+ " \"file:///media/hdd1/code/sigspatial-2021-cafe-analysis/data/seattle_neighborhoods.csv\"\n",
+ " )\n",
+ " # transform the geometry column into a Geometry-type\n",
+ " .withColumn(\"geometry\", f.expr(\"ST_GeomFromWkt(geometry)\"))\n",
")\n",
- "WA_neighbs.createOrReplaceTempView('WA_neighbs')\n",
+ "WA_neighbs.createOrReplaceTempView(\"WA_neighbs\")\n",
"\n",
"WA_neighbs.limit(10).toPandas().head()"
]
@@ -2051,8 +2098,13 @@
},
"outputs": [],
"source": [
- "cafes_geo = cafes_latest.withColumn('cafe_geometry', f.expr(\"ST_Point(CAST(longitude AS Decimal(24,20)), CAST(latitude AS Decimal(24,20)))\")).select('placekey', 'cafe_geometry')\n",
- "cafes_geo.createOrReplaceTempView('cafes_geo')"
+ "cafes_geo = cafes_latest.withColumn(\n",
+ " \"cafe_geometry\",\n",
+ " f.expr(\n",
+ " \"ST_Point(CAST(longitude AS Decimal(24,20)), CAST(latitude AS Decimal(24,20)))\"\n",
+ " ),\n",
+ ").select(\"placekey\", \"cafe_geometry\")\n",
+ "cafes_geo.createOrReplaceTempView(\"cafes_geo\")"
]
},
{
@@ -2069,11 +2121,11 @@
"outputs": [],
"source": [
"# perform a spatial join\n",
- "q = '''\n",
+ "q = \"\"\"\n",
"SELECT cafes_geo.placekey, WA_neighbs.S_HOOD as neighborhood, WA_neighbs.geometry\n",
"FROM WA_neighbs, cafes_geo\n",
"WHERE ST_Intersects(WA_neighbs.geometry, cafes_geo.cafe_geometry)\n",
- "'''\n",
+ "\"\"\"\n",
"\n",
"cafe_neighb_join = spark.sql(q)"
]
@@ -2199,13 +2251,12 @@
"source": [
"# add the visit and visitor counts and aggregate up to the neighborhood\n",
"neighborhood_agg = (\n",
- " cafe_neighb_join\n",
- " .join(total_visits, 'placekey')\n",
- " .groupBy('neighborhood', f.col('geometry').cast('string').alias('geometry'))\n",
- " .agg(\n",
- " f.sum('total_visits').alias('total_visits'),\n",
- " f.sum('total_visitors').alias('total_visitors')\n",
- " )\n",
+ " cafe_neighb_join.join(total_visits, \"placekey\")\n",
+ " .groupBy(\"neighborhood\", f.col(\"geometry\").cast(\"string\").alias(\"geometry\"))\n",
+ " .agg(\n",
+ " f.sum(\"total_visits\").alias(\"total_visits\"),\n",
+ " f.sum(\"total_visitors\").alias(\"total_visitors\"),\n",
+ " )\n",
")"
]
},
@@ -2231,11 +2282,12 @@
}
],
"source": [
- "neighbs_gdf = (\n",
- " neighborhood_agg\n",
- " .toPandas()\n",
- ")\n",
- "neighbs_gdf = gpd.GeoDataFrame(neighbs_gdf, geometry = gpd.GeoSeries.from_wkt(neighbs_gdf['geometry']), crs = 'EPSG:4326')"
+ "neighbs_gdf = neighborhood_agg.toPandas()\n",
+ "neighbs_gdf = gpd.GeoDataFrame(\n",
+ " neighbs_gdf,\n",
+ " geometry=gpd.GeoSeries.from_wkt(neighbs_gdf[\"geometry\"]),\n",
+ " crs=\"EPSG:4326\",\n",
+ ")"
]
},
{
@@ -2252,38 +2304,37 @@
"outputs": [],
"source": [
"def map_neighbs(gdf):\n",
- " \n",
- " # map bounds\n",
- " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
- " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
- " folium_bounds = [sw, ne]\n",
- " \n",
- " # map\n",
- " x = gdf.centroid.x[0]\n",
- " y = gdf.centroid.y[0]\n",
- " \n",
- " map_ = folium.Map(\n",
- " location = [y, x],\n",
- " tiles = \"OpenStreetMap\"\n",
- " )\n",
- " \n",
- " gdf['percentile'] = pd.qcut(gdf['total_visits'], 100, labels=False) / 100\n",
- " \n",
- " folium.GeoJson(\n",
- " gdf[['neighborhood', 'total_visits', 'total_visitors', 'percentile', 'geometry']],\n",
- " style_function = lambda x: {\n",
- " 'weight':0,\n",
- " 'color':'blue',\n",
- " 'fillOpacity': x['properties']['percentile']\n",
- " },\n",
- " tooltip = folium.features.GeoJsonTooltip(\n",
- " fields = ['neighborhood', 'total_visits', 'total_visitors', 'percentile']\n",
- " )\n",
+ "\n",
+ " # map bounds\n",
+ " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
+ " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
+ " folium_bounds = [sw, ne]\n",
+ "\n",
+ " # map\n",
+ " x = gdf.centroid.x[0]\n",
+ " y = gdf.centroid.y[0]\n",
+ "\n",
+ " map_ = folium.Map(location=[y, x], tiles=\"OpenStreetMap\")\n",
+ "\n",
+ " gdf[\"percentile\"] = pd.qcut(gdf[\"total_visits\"], 100, labels=False) / 100\n",
+ "\n",
+ " folium.GeoJson(\n",
+ " gdf[\n",
+ " [\"neighborhood\", \"total_visits\", \"total_visitors\", \"percentile\", \"geometry\"]\n",
+ " ],\n",
+ " style_function=lambda x: {\n",
+ " \"weight\": 0,\n",
+ " \"color\": \"blue\",\n",
+ " \"fillOpacity\": x[\"properties\"][\"percentile\"],\n",
+ " },\n",
+ " tooltip=folium.features.GeoJsonTooltip(\n",
+ " fields=[\"neighborhood\", \"total_visits\", \"total_visitors\", \"percentile\"]\n",
+ " ),\n",
" ).add_to(map_)\n",
"\n",
- " map_.fit_bounds(folium_bounds) \n",
- " \n",
- " return map_"
+ " map_.fit_bounds(folium_bounds)\n",
+ "\n",
+ " return map_"
]
},
{
@@ -2356,13 +2407,12 @@
"outputs": [],
"source": [
"home_loc_most_cafe_visitors = (\n",
- " sample\n",
- " .select(f.explode('visitor_home_cbgs'))\n",
- " .withColumnRenamed('key', 'cbg')\n",
- " .withColumnRenamed('value', 'visitors')\n",
- " .groupBy('cbg')\n",
- " .agg(f.sum('visitors').alias('visitors'))\n",
- " .orderBy('visitors', ascending = False)\n",
+ " sample.select(f.explode(\"visitor_home_cbgs\"))\n",
+ " .withColumnRenamed(\"key\", \"cbg\")\n",
+ " .withColumnRenamed(\"value\", \"visitors\")\n",
+ " .groupBy(\"cbg\")\n",
+ " .agg(f.sum(\"visitors\").alias(\"visitors\"))\n",
+ " .orderBy(\"visitors\", ascending=False)\n",
")"
]
},
@@ -2498,15 +2548,15 @@
"source": [
"# Map of the top 1000 CBGs in terms of visitors' origins\n",
"home_loc_gdf = (\n",
- " home_loc_most_cafe_visitors\n",
- " .limit(1000)\n",
- " .join(\n",
- " WA_cbgs.select('cbg', 'cbg_polygon_geometry'),\n",
- " 'cbg'\n",
- " )\n",
- " .toPandas()\n",
+ " home_loc_most_cafe_visitors.limit(1000)\n",
+ " .join(WA_cbgs.select(\"cbg\", \"cbg_polygon_geometry\"), \"cbg\")\n",
+ " .toPandas()\n",
")\n",
- "home_loc_gdf = gpd.GeoDataFrame(home_loc_gdf, geometry = gpd.GeoSeries.from_wkt(home_loc_gdf['cbg_polygon_geometry']), crs = 'EPSG:4326')"
+ "home_loc_gdf = gpd.GeoDataFrame(\n",
+ " home_loc_gdf,\n",
+ " geometry=gpd.GeoSeries.from_wkt(home_loc_gdf[\"cbg_polygon_geometry\"]),\n",
+ " crs=\"EPSG:4326\",\n",
+ ")"
]
},
{
@@ -2523,38 +2573,33 @@
"outputs": [],
"source": [
"def map_cbgs(gdf):\n",
- " \n",
- " # map bounds\n",
- " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
- " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
- " folium_bounds = [sw, ne]\n",
- " \n",
- " # map\n",
- " x = gdf.centroid.x[0]\n",
- " y = gdf.centroid.y[0]\n",
- " \n",
- " map_ = folium.Map(\n",
- " location = [y, x],\n",
- " tiles = \"OpenStreetMap\"\n",
- " )\n",
- " \n",
- " gdf['quantile'] = pd.qcut(gdf['visitors'], 100, labels=False) / 100\n",
- " \n",
- " folium.GeoJson(\n",
- " gdf[['cbg', 'visitors', 'geometry', 'quantile']],\n",
- " style_function = lambda x: {\n",
- " 'weight':0,\n",
- " 'color':'blue',\n",
- " 'fillOpacity': x['properties']['quantile']\n",
- " },\n",
- " tooltip = folium.features.GeoJsonTooltip(\n",
- " fields = ['cbg', 'visitors', 'quantile']\n",
- " )\n",
+ "\n",
+ " # map bounds\n",
+ " sw = [gdf.unary_union.bounds[1], gdf.unary_union.bounds[0]]\n",
+ " ne = [gdf.unary_union.bounds[3], gdf.unary_union.bounds[2]]\n",
+ " folium_bounds = [sw, ne]\n",
+ "\n",
+ " # map\n",
+ " x = gdf.centroid.x[0]\n",
+ " y = gdf.centroid.y[0]\n",
+ "\n",
+ " map_ = folium.Map(location=[y, x], tiles=\"OpenStreetMap\")\n",
+ "\n",
+ " gdf[\"quantile\"] = pd.qcut(gdf[\"visitors\"], 100, labels=False) / 100\n",
+ "\n",
+ " folium.GeoJson(\n",
+ " gdf[[\"cbg\", \"visitors\", \"geometry\", \"quantile\"]],\n",
+ " style_function=lambda x: {\n",
+ " \"weight\": 0,\n",
+ " \"color\": \"blue\",\n",
+ " \"fillOpacity\": x[\"properties\"][\"quantile\"],\n",
+ " },\n",
+ " tooltip=folium.features.GeoJsonTooltip(fields=[\"cbg\", \"visitors\", \"quantile\"]),\n",
" ).add_to(map_)\n",
"\n",
- " map_.fit_bounds(folium_bounds) \n",
- " \n",
- " return map_"
+ " map_.fit_bounds(folium_bounds)\n",
+ "\n",
+ " return map_"
]
},
{
diff --git a/docs/usecases/utilities.py b/docs/usecases/utilities.py
index 6e06bbb77..047e75f3a 100644
--- a/docs/usecases/utilities.py
+++ b/docs/usecases/utilities.py
@@ -15,99 +15,142 @@
# specific language governing permissions and limitations
# under the License.
+
def getConfig():
- config = {'version': 'v1',
- 'config': {'visState': {'filters': [],
- 'layers': [{'id': 'ikzru0t',
- 'type': 'geojson',
- 'config': {'dataId': 'AirportCount',
- 'label': 'AirportCount',
- 'color': [218, 112, 191],
- 'highlightColor': [252, 242, 26, 255],
- 'columns': {'geojson': 'geometry'},
- 'isVisible': True,
- 'visConfig': {'opacity': 0.8,
- 'strokeOpacity': 0.8,
- 'thickness': 0.5,
- 'strokeColor': [18, 92, 119],
- 'colorRange': {'name': 'Uber Viz Sequential 6',
- 'type': 'sequential',
- 'category': 'Uber',
- 'colors': ['#E6FAFA',
- '#C1E5E6',
- '#9DD0D4',
- '#75BBC1',
- '#4BA7AF',
- '#00939C',
- '#108188',
- '#0E7077']},
- 'strokeColorRange': {'name': 'Global Warming',
- 'type': 'sequential',
- 'category': 'Uber',
- 'colors': ['#5A1846',
- '#900C3F',
- '#C70039',
- '#E3611C',
- '#F1920E',
- '#FFC300']},
- 'radius': 10,
- 'sizeRange': [0, 10],
- 'radiusRange': [0, 50],
- 'heightRange': [0, 500],
- 'elevationScale': 5,
- 'enableElevationZoomFactor': True,
- 'stroked': False,
- 'filled': True,
- 'enable3d': False,
- 'wireframe': False},
- 'hidden': False,
- 'textLabel': [{'field': None,
- 'color': [255, 255, 255],
- 'size': 18,
- 'offset': [0, 0],
- 'anchor': 'start',
- 'alignment': 'center'}]},
- 'visualChannels': {'colorField': {'name': 'AirportCount',
- 'type': 'integer'},
- 'colorScale': 'quantize',
- 'strokeColorField': None,
- 'strokeColorScale': 'quantile',
- 'sizeField': None,
- 'sizeScale': 'linear',
- 'heightField': None,
- 'heightScale': 'linear',
- 'radiusField': None,
- 'radiusScale': 'linear'}}],
- 'interactionConfig': {'tooltip': {'fieldsToShow': {'AirportCount': [{'name': 'NAME_EN',
- 'format': None},
- {'name': 'AirportCount', 'format': None}]},
- 'compareMode': False,
- 'compareType': 'absolute',
- 'enabled': True},
- 'brush': {'size': 0.5, 'enabled': False},
- 'geocoder': {'enabled': False},
- 'coordinate': {'enabled': False}},
- 'layerBlending': 'normal',
- 'splitMaps': [],
- 'animationConfig': {'currentTime': None, 'speed': 1}},
- 'mapState': {'bearing': 0,
- 'dragRotate': False,
- 'latitude': 56.422456606624316,
- 'longitude': 9.778836615231771,
- 'pitch': 0,
- 'zoom': 0.4214991225736964,
- 'isSplit': False},
- 'mapStyle': {'styleType': 'dark',
- 'topLayerGroups': {},
- 'visibleLayerGroups': {'label': True,
- 'road': True,
- 'border': False,
- 'building': True,
- 'water': True,
- 'land': True,
- '3d building': False},
- 'threeDBuildingColor': [9.665468314072013,
- 17.18305478057247,
- 31.1442867897876],
- 'mapStyles': {}}}}
+ config = {
+ "version": "v1",
+ "config": {
+ "visState": {
+ "filters": [],
+ "layers": [
+ {
+ "id": "ikzru0t",
+ "type": "geojson",
+ "config": {
+ "dataId": "AirportCount",
+ "label": "AirportCount",
+ "color": [218, 112, 191],
+ "highlightColor": [252, 242, 26, 255],
+ "columns": {"geojson": "geometry"},
+ "isVisible": True,
+ "visConfig": {
+ "opacity": 0.8,
+ "strokeOpacity": 0.8,
+ "thickness": 0.5,
+ "strokeColor": [18, 92, 119],
+ "colorRange": {
+ "name": "Uber Viz Sequential 6",
+ "type": "sequential",
+ "category": "Uber",
+ "colors": [
+ "#E6FAFA",
+ "#C1E5E6",
+ "#9DD0D4",
+ "#75BBC1",
+ "#4BA7AF",
+ "#00939C",
+ "#108188",
+ "#0E7077",
+ ],
+ },
+ "strokeColorRange": {
+ "name": "Global Warming",
+ "type": "sequential",
+ "category": "Uber",
+ "colors": [
+ "#5A1846",
+ "#900C3F",
+ "#C70039",
+ "#E3611C",
+ "#F1920E",
+ "#FFC300",
+ ],
+ },
+ "radius": 10,
+ "sizeRange": [0, 10],
+ "radiusRange": [0, 50],
+ "heightRange": [0, 500],
+ "elevationScale": 5,
+ "enableElevationZoomFactor": True,
+ "stroked": False,
+ "filled": True,
+ "enable3d": False,
+ "wireframe": False,
+ },
+ "hidden": False,
+ "textLabel": [
+ {
+ "field": None,
+ "color": [255, 255, 255],
+ "size": 18,
+ "offset": [0, 0],
+ "anchor": "start",
+ "alignment": "center",
+ }
+ ],
+ },
+ "visualChannels": {
+ "colorField": {"name": "AirportCount", "type": "integer"},
+ "colorScale": "quantize",
+ "strokeColorField": None,
+ "strokeColorScale": "quantile",
+ "sizeField": None,
+ "sizeScale": "linear",
+ "heightField": None,
+ "heightScale": "linear",
+ "radiusField": None,
+ "radiusScale": "linear",
+ },
+ }
+ ],
+ "interactionConfig": {
+ "tooltip": {
+ "fieldsToShow": {
+ "AirportCount": [
+ {"name": "NAME_EN", "format": None},
+ {"name": "AirportCount", "format": None},
+ ]
+ },
+ "compareMode": False,
+ "compareType": "absolute",
+ "enabled": True,
+ },
+ "brush": {"size": 0.5, "enabled": False},
+ "geocoder": {"enabled": False},
+ "coordinate": {"enabled": False},
+ },
+ "layerBlending": "normal",
+ "splitMaps": [],
+ "animationConfig": {"currentTime": None, "speed": 1},
+ },
+ "mapState": {
+ "bearing": 0,
+ "dragRotate": False,
+ "latitude": 56.422456606624316,
+ "longitude": 9.778836615231771,
+ "pitch": 0,
+ "zoom": 0.4214991225736964,
+ "isSplit": False,
+ },
+ "mapStyle": {
+ "styleType": "dark",
+ "topLayerGroups": {},
+ "visibleLayerGroups": {
+ "label": True,
+ "road": True,
+ "border": False,
+ "building": True,
+ "water": True,
+ "land": True,
+ "3d building": False,
+ },
+ "threeDBuildingColor": [
+ 9.665468314072013,
+ 17.18305478057247,
+ 31.1442867897876,
+ ],
+ "mapStyles": {},
+ },
+ },
+ }
return config
diff --git a/python/sedona/core/SpatialRDD/__init__.py b/python/sedona/core/SpatialRDD/__init__.py
index e51e5470e..c39df7835 100644
--- a/python/sedona/core/SpatialRDD/__init__.py
+++ b/python/sedona/core/SpatialRDD/__init__.py
@@ -24,5 +24,10 @@ from .rectangle_rdd import SpatialRDD
__all__ = [
- "PolygonRDD", "PointRDD", "CircleRDD", "LineStringRDD", "RectangleRDD", "SpatialRDD"
+ "PolygonRDD",
+ "PointRDD",
+ "CircleRDD",
+ "LineStringRDD",
+ "RectangleRDD",
+ "SpatialRDD",
]
diff --git a/python/sedona/core/SpatialRDD/circle_rdd.py b/python/sedona/core/SpatialRDD/circle_rdd.py
index ea98534d6..48142a52a 100644
--- a/python/sedona/core/SpatialRDD/circle_rdd.py
+++ b/python/sedona/core/SpatialRDD/circle_rdd.py
@@ -29,34 +29,35 @@ class CircleRDD(SpatialRDD, metaclass=MultipleMeta):
:param Radius: float
"""
super()._do_init(spatialRDD._sc)
- self._srdd = self._jvm_spatial_rdd(
- spatialRDD._srdd,
- Radius
- )
+ self._srdd = self._jvm_spatial_rdd(spatialRDD._srdd, Radius)
- def getCenterPointAsSpatialRDD(self) -> 'PointRDD':
+ def getCenterPointAsSpatialRDD(self) -> "PointRDD":
from sedona.core.SpatialRDD import PointRDD
+
srdd = self._srdd.getCenterPointAsSpatialRDD()
point_rdd = PointRDD()
point_rdd.set_srdd(srdd)
return point_rdd
- def getCenterPolygonAsSpatialRDD(self) -> 'PolygonRDD':
+ def getCenterPolygonAsSpatialRDD(self) -> "PolygonRDD":
from sedona.core.SpatialRDD import PolygonRDD
+
srdd = self._srdd.getCenterPolygonAsSpatialRDD()
polygon_rdd = PolygonRDD()
polygon_rdd.set_srdd(srdd)
return polygon_rdd
- def getCenterLineStringRDDAsSpatialRDD(self) -> 'LineStringRDD':
+ def getCenterLineStringRDDAsSpatialRDD(self) -> "LineStringRDD":
from sedona.core.SpatialRDD import LineStringRDD
+
srdd = self._srdd.getCenterPolygonAsSpatialRDD()
linestring_rdd = LineStringRDD()
linestring_rdd.set_srdd(srdd)
return linestring_rdd
- def getCenterRectangleRDDAsSpatialRDD(self) -> 'RectangleRDD':
+ def getCenterRectangleRDDAsSpatialRDD(self) -> "RectangleRDD":
from sedona.core.SpatialRDD import RectangleRDD
+
srdd = self._srdd.getCenterLineStringRDDAsSpatialRDD()
rectangle_rdd = RectangleRDD()
rectangle_rdd.set_srdd(srdd)
diff --git a/python/sedona/core/SpatialRDD/linestring_rdd.py b/python/sedona/core/SpatialRDD/linestring_rdd.py
index 0bd720f56..63df1587c 100644
--- a/python/sedona/core/SpatialRDD/linestring_rdd.py
+++ b/python/sedona/core/SpatialRDD/linestring_rdd.py
@@ -30,7 +30,9 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
def __init__(self, rdd: RDD):
super().__init__(rdd.ctx)
- spatial_rdd = PythonRddToJavaRDDAdapter(self._jvm).deserialize_to_linestring_raw_rdd(rdd._jrdd)
+ spatial_rdd = PythonRddToJavaRDDAdapter(
+ self._jvm
+ ).deserialize_to_linestring_raw_rdd(rdd._jrdd)
srdd = self._jvm_spatial_rdd(spatial_rdd)
self._srdd = srdd
@@ -48,8 +50,16 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
jsrdd = rawSpatialRDD.jsrdd
self._srdd = self._jvm_spatial_rdd(jsrdd)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, startOffset: int, endOffset: int,
- splitter: FileDataSplitter, carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ startOffset: int,
+ endOffset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -70,11 +80,18 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
endOffset,
jvm_splitter,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, startOffset: int, endOffset: int,
- splitter: FileDataSplitter, carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ startOffset: int,
+ endOffset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -96,8 +113,14 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
carryInputData,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter, carryInputData: bool,
- partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -110,15 +133,16 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
super().__init__(sparkContext)
jvm_splitter = FileSplitterJvm(self._jvm, splitter).jvm_instance
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter,
- carryInputData,
- partitions
+ self._jsc, InputLocation, jvm_splitter, carryInputData, partitions
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -130,10 +154,7 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
super().__init__(sparkContext)
jvm_splitter = FileSplitterJvm(self._jvm, splitter).jvm_instance
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter,
- carryInputData
+ self._jsc, InputLocation, jvm_splitter, carryInputData
)
@property
@@ -146,6 +167,7 @@ class LineStringRDD(SpatialRDD, metaclass=MultipleMeta):
def MinimumBoundingRectangle(self):
from sedona.core.SpatialRDD import RectangleRDD
+
rectangle_rdd = RectangleRDD()
srdd = self._srdd.MinimumBoundingRectangle()
diff --git a/python/sedona/core/SpatialRDD/point_rdd.py b/python/sedona/core/SpatialRDD/point_rdd.py
index 658e59217..dbe401db6 100644
--- a/python/sedona/core/SpatialRDD/point_rdd.py
+++ b/python/sedona/core/SpatialRDD/point_rdd.py
@@ -33,7 +33,9 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
"""
super().__init__(rdd.ctx)
- spatial_rdd = PythonRddToJavaRDDAdapter(self._jvm).deserialize_to_point_raw_rdd(rdd._jrdd)
+ spatial_rdd = PythonRddToJavaRDDAdapter(self._jvm).deserialize_to_point_raw_rdd(
+ rdd._jrdd
+ )
srdd = self._jvm_spatial_rdd(spatial_rdd)
self._srdd = srdd
@@ -51,8 +53,15 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
jsrdd = rawSpatialRDD.jsrdd
self._srdd = self._jvm_spatial_rdd(jsrdd)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, Offset: int, splitter: FileDataSplitter,
- carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ Offset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -71,11 +80,17 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
Offset,
jvm_splitter,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, Offset: int, splitter: FileDataSplitter,
- carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ Offset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -87,15 +102,17 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
super().__init__(sparkContext)
jvm_splitter = FileSplitterJvm(self._jvm, splitter).jvm_instance
self._srdd = self._jvm_spatial_rdd(
- sparkContext._jsc,
- InputLocation,
- Offset,
- jvm_splitter,
- carryInputData
+ sparkContext._jsc, InputLocation, Offset, jvm_splitter, carryInputData
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter, carryInputData: bool,
- partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -107,15 +124,16 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
super().__init__(sparkContext)
jvm_splitter = FileSplitterJvm(self._jvm, splitter).jvm_instance
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter,
- carryInputData,
- partitions
+ self._jsc, InputLocation, jvm_splitter, carryInputData, partitions
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext instance
@@ -127,10 +145,7 @@ class PointRDD(SpatialRDD, metaclass=MultipleMeta):
super().__init__(sparkContext)
jvm_splitter = FileSplitterJvm(self._jvm, splitter).jvm_instance
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter,
- carryInputData
+ self._jsc, InputLocation, jvm_splitter, carryInputData
)
def MinimumBoundingRectangle(self):
diff --git a/python/sedona/core/SpatialRDD/polygon_rdd.py b/python/sedona/core/SpatialRDD/polygon_rdd.py
index c72ba9c72..072776980 100644
--- a/python/sedona/core/SpatialRDD/polygon_rdd.py
+++ b/python/sedona/core/SpatialRDD/polygon_rdd.py
@@ -28,7 +28,9 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
def __init__(self, rdd: RDD):
super().__init__(rdd.ctx)
- spatial_rdd = PythonRddToJavaRDDAdapter(self._jvm).deserialize_to_polygon_raw_rdd(rdd._jrdd)
+ spatial_rdd = PythonRddToJavaRDDAdapter(
+ self._jvm
+ ).deserialize_to_polygon_raw_rdd(rdd._jrdd)
srdd = self._jvm_spatial_rdd(spatial_rdd)
self._srdd = srdd
@@ -45,8 +47,16 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
jsrdd = rawSpatialRDD.jsrdd
self._srdd = self._jvm_spatial_rdd(jsrdd)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, startOffset: int, endOffset: int,
- splitter: FileDataSplitter, carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ startOffset: int,
+ endOffset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -67,11 +77,18 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
endOffset,
jvm_splitter.jvm_instance,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, startOffset: int, endOffset: int,
- splitter: FileDataSplitter, carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ startOffset: int,
+ endOffset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -90,11 +107,17 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
startOffset,
endOffset,
jvm_splitter.jvm_instance,
- carryInputData
+ carryInputData,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -112,11 +135,16 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
InputLocation,
jvm_splitter.jvm_instance,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -129,14 +157,12 @@ class PolygonRDD(SpatialRDD, metaclass=MultipleMeta):
jvm_splitter = FileSplitterJvm(self._jvm, splitter)
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter.jvm_instance,
- carryInputData
+ self._jsc, InputLocation, jvm_splitter.jvm_instance, carryInputData
)
def MinimumBoundingRectangle(self):
from sedona.core.SpatialRDD import RectangleRDD
+
rectangle_rdd = RectangleRDD()
srdd = self._srdd.MinimumBoundingRectangle()
diff --git a/python/sedona/core/SpatialRDD/rectangle_rdd.py b/python/sedona/core/SpatialRDD/rectangle_rdd.py
index 48d9a222f..d5de40d5b 100644
--- a/python/sedona/core/SpatialRDD/rectangle_rdd.py
+++ b/python/sedona/core/SpatialRDD/rectangle_rdd.py
@@ -39,8 +39,15 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
jsrdd = rawSpatialRDD.jsrdd
self._srdd = self._jvm_spatial_rdd(jsrdd)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, Offset: int,
- splitter: FileDataSplitter, carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ Offset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -59,11 +66,17 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
Offset,
jvm_splitter.jvm_instance,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, Offset: int,
- splitter: FileDataSplitter, carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ Offset: int,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -76,15 +89,17 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
jvm_splitter = FileSplitterJvm(self._jvm, splitter)
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- Offset,
- jvm_splitter.jvm_instance,
- carryInputData
+ self._jsc, InputLocation, Offset, jvm_splitter.jvm_instance, carryInputData
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool, partitions: int):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ partitions: int,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -102,11 +117,16 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
InputLocation,
jvm_splitter.jvm_instance,
carryInputData,
- partitions
+ partitions,
)
- def __init__(self, sparkContext: SparkContext, InputLocation: str, splitter: FileDataSplitter,
- carryInputData: bool):
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ InputLocation: str,
+ splitter: FileDataSplitter,
+ carryInputData: bool,
+ ):
"""
:param sparkContext: SparkContext, the spark context
@@ -119,11 +139,9 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
jvm_splitter = FileSplitterJvm(self._jvm, splitter)
self._srdd = self._jvm_spatial_rdd(
- self._jsc,
- InputLocation,
- jvm_splitter.jvm_instance,
- carryInputData
+ self._jsc, InputLocation, jvm_splitter.jvm_instance, carryInputData
)
+
@property
def _jvm_spatial_rdd(self):
spatial_factory = SpatialRDDFactory(self._sc)
@@ -133,4 +151,6 @@ class RectangleRDD(SpatialRDD, metaclass=MultipleMeta):
return jvm_polygon_rdd
def MinimumBoundingRectangle(self):
- raise NotImplementedError("RectangleRDD has not MinimumBoundingRectangle method.")
+ raise NotImplementedError(
+ "RectangleRDD has not MinimumBoundingRectangle method."
+ )
diff --git a/python/sedona/core/SpatialRDD/spatial_rdd.py b/python/sedona/core/SpatialRDD/spatial_rdd.py
index cc0fbe6e4..5b3376437 100644
--- a/python/sedona/core/SpatialRDD/spatial_rdd.py
+++ b/python/sedona/core/SpatialRDD/spatial_rdd.py
@@ -42,7 +42,7 @@ class SpatialPartitioner:
jvm_partitioner = attr.ib()
@classmethod
- def from_java_class_name(cls, jvm_partitioner) -> 'SpatialPartitioner':
+ def from_java_class_name(cls, jvm_partitioner) -> "SpatialPartitioner":
if jvm_partitioner is not None:
jvm_full_name = jvm_partitioner.toString()
full_class_name = jvm_full_name.split("@")[0]
@@ -63,7 +63,9 @@ class JvmSpatialRDD:
self.jsrdd.saveAsObjectFile(location)
def persist(self, storage_level: StorageLevel):
- new_jsrdd = self.jsrdd.persist(JvmStorageLevel(self.sc._jvm, storage_level).jvm_instance)
+ new_jsrdd = self.jsrdd.persist(
+ JvmStorageLevel(self.sc._jvm, storage_level).jvm_instance
+ )
self.jsrdd = new_jsrdd
def count(self):
@@ -155,7 +157,9 @@ class SpatialRDD:
java_boundary_envelope = get_field(self._srdd, "boundaryEnvelope")
return Envelope.from_jvm_instance(java_boundary_envelope)
- def buildIndex(self, indexType: Union[str, IndexType], buildIndexOnSpatialPartitionedRDD: bool) -> bool:
+ def buildIndex(
+ self, indexType: Union[str, IndexType], buildIndexOnSpatialPartitionedRDD: bool
+ ) -> bool:
"""
:param indexType:
@@ -171,8 +175,7 @@ class SpatialRDD:
else:
raise TypeError("indexType should be str or IndexType")
return self._srdd.buildIndex(
- index_type.jvm_instance,
- buildIndexOnSpatialPartitionedRDD
+ index_type.jvm_instance, buildIndexOnSpatialPartitionedRDD
)
else:
raise AttributeError("Please run spatial partitioning before")
@@ -224,12 +227,17 @@ class SpatialRDD:
:return:
"""
- serialized_spatial_rdd = SedonaPythonConverter(self._jvm).translate_spatial_rdd_to_python(
- self._srdd.getRawSpatialRDD())
+ serialized_spatial_rdd = SedonaPythonConverter(
+ self._jvm
+ ).translate_spatial_rdd_to_python(self._srdd.getRawSpatialRDD())
if not hasattr(self, "_raw_spatial_rdd"):
RDD.saveAsObjectFile = lambda x, path: x._jrdd.saveAsObjectFile(path)
- setattr(self, "_raw_spatial_rdd", RDD(serialized_spatial_rdd, self._sc, SedonaPickler()))
+ setattr(
+ self,
+ "_raw_spatial_rdd",
+ RDD(serialized_spatial_rdd, self._sc, SedonaPickler()),
+ )
else:
self._raw_spatial_rdd._jrdd = serialized_spatial_rdd
@@ -269,7 +277,10 @@ class SpatialRDD:
if jvm_grids:
number_of_grids = jvm_grids.size()
- envelopes = [Envelope.from_jvm_instance(jvm_grids[index]) for index in range(number_of_grids)]
+ envelopes = [
+ Envelope.from_jvm_instance(jvm_grids[index])
+ for index in range(number_of_grids)
+ ]
return envelopes
else:
@@ -338,7 +349,9 @@ class SpatialRDD:
self._jvm = spatial_rdd._jvm
self._spatial_partitioned = spatial_rdd._spatial_partitioned
elif isinstance(spatial_rdd, RDD):
- jrdd = JvmSedonaPythonConverter(self._jvm).translate_python_rdd_to_java(spatial_rdd._jrdd)
+ jrdd = JvmSedonaPythonConverter(self._jvm).translate_python_rdd_to_java(
+ spatial_rdd._jrdd
+ )
self._srdd.setRawSpatialRDD(jrdd)
else:
self._srdd.setRawSpatialRDD(spatial_rdd)
@@ -387,18 +400,28 @@ class SpatialRDD:
:return:
"""
- serialized_spatial_rdd = SedonaPythonConverter(self._jvm).translate_spatial_rdd_to_python(
- get_field(self._srdd, "spatialPartitionedRDD"))
+ serialized_spatial_rdd = SedonaPythonConverter(
+ self._jvm
+ ).translate_spatial_rdd_to_python(
+ get_field(self._srdd, "spatialPartitionedRDD")
+ )
if not hasattr(self, "_spatial_partitioned_rdd"):
- setattr(self, "_spatial_partitioned_rdd", RDD(serialized_spatial_rdd, self._sc, SedonaPickler()))
+ setattr(
+ self,
+ "_spatial_partitioned_rdd",
+ RDD(serialized_spatial_rdd, self._sc, SedonaPickler()),
+ )
else:
self._spatial_partitioned_rdd._jrdd = serialized_spatial_rdd
return getattr(self, "_spatial_partitioned_rdd")
- def spatialPartitioning(self, partitioning: Union[str, GridType, SpatialPartitioner, List[Envelope]],
- num_partitions: Optional[int] = None) -> bool:
+ def spatialPartitioning(
+ self,
+ partitioning: Union[str, GridType, SpatialPartitioner, List[Envelope]],
+ num_partitions: Optional[int] = None,
+ ) -> bool:
"""
:param partitioning: partitioning type
@@ -435,7 +458,11 @@ class SpatialRDD:
return self._srdd
def getRawJvmSpatialRDD(self) -> JvmSpatialRDD:
- return JvmSpatialRDD(jsrdd=self._srdd.getRawSpatialRDD(), sc=self._sc, tp=SpatialType.from_str(self.name))
+ return JvmSpatialRDD(
+ jsrdd=self._srdd.getRawSpatialRDD(),
+ sc=self._sc,
+ tp=SpatialType.from_str(self.name),
+ )
@property
def rawJvmSpatialRDD(self) -> JvmSpatialRDD:
@@ -444,7 +471,9 @@ class SpatialRDD:
@rawJvmSpatialRDD.setter
def rawJvmSpatialRDD(self, jsrdd_p: JvmSpatialRDD):
if jsrdd_p.tp.value.lower() != self.name:
- raise TypeError(f"value should be type {self.name} but {jsrdd_p.tp} was found")
+ raise TypeError(
+ f"value should be type {self.name} but {jsrdd_p.tp} was found"
+ )
self._sc = jsrdd_p.sc
self._jvm = self._sc._jvm
@@ -452,8 +481,10 @@ class SpatialRDD:
self.setRawSpatialRDD(jsrdd_p.jsrdd)
def getJvmSpatialPartitionedRDD(self) -> JvmSpatialRDD:
- return JvmSpatialRDD(jsrdd=get_field(
- self._srdd, "spatialPartitionedRDD"), sc=self._sc, tp=SpatialType.from_str(self.name)
+ return JvmSpatialRDD(
+ jsrdd=get_field(self._srdd, "spatialPartitionedRDD"),
+ sc=self._sc,
+ tp=SpatialType.from_str(self.name),
)
@property
@@ -463,7 +494,9 @@ class SpatialRDD:
@jvmSpatialPartitionedRDD.setter
def jvmSpatialPartitionedRDD(self, jsrdd_p: JvmSpatialRDD):
if jsrdd_p.tp.value.lower() != self.name:
- raise TypeError(f"value should be type {self.name} but {jsrdd_p.tp} was found")
+ raise TypeError(
+ f"value should be type {self.name} but {jsrdd_p.tp} was found"
+ )
self._sc = jsrdd_p.sc
self._jvm = self._sc._jvm
diff --git a/python/sedona/core/enums/file_data_splitter.py b/python/sedona/core/enums/file_data_splitter.py
index d0ecdbe41..81b402b3f 100644
--- a/python/sedona/core/enums/file_data_splitter.py
+++ b/python/sedona/core/enums/file_data_splitter.py
@@ -47,7 +47,11 @@ class FileSplitterJvm(JvmObject):
splitter = attr.ib(type=FileDataSplitter)
def _create_jvm_instance(self):
- return self.jvm_splitter(self.splitter.value) if self.splitter.value is not None else None
+ return (
+ self.jvm_splitter(self.splitter.value)
+ if self.splitter.value is not None
+ else None
+ )
@property
@require(["FileDataSplitter"])
diff --git a/python/sedona/core/enums/grid_type.py b/python/sedona/core/enums/grid_type.py
index b5d816b9e..89deeee96 100644
--- a/python/sedona/core/enums/grid_type.py
+++ b/python/sedona/core/enums/grid_type.py
@@ -28,7 +28,7 @@ class GridType(Enum):
KDBTREE = "KDBTREE"
@classmethod
- def from_str(cls, grid: str) -> 'GridType':
+ def from_str(cls, grid: str) -> "GridType":
try:
grid = getattr(cls, grid.upper())
except AttributeError:
diff --git a/python/sedona/core/enums/index_type.py b/python/sedona/core/enums/index_type.py
index 290f43a02..3a00004d8 100644
--- a/python/sedona/core/enums/index_type.py
+++ b/python/sedona/core/enums/index_type.py
@@ -41,7 +41,11 @@ class IndexTypeJvm(JvmObject):
index_type = attr.ib(type=IndexType)
def _create_jvm_instance(self):
- return self.jvm_index(self.index_type.value) if self.index_type.value is not None else None
+ return (
+ self.jvm_index(self.index_type.value)
+ if self.index_type.value is not None
+ else None
+ )
@property
@require(["FileDataSplitter"])
diff --git a/python/sedona/core/enums/spatial.py b/python/sedona/core/enums/spatial.py
index f8c6fd52c..da92e374f 100644
--- a/python/sedona/core/enums/spatial.py
+++ b/python/sedona/core/enums/spatial.py
@@ -27,7 +27,7 @@ class SpatialType(Enum):
CIRCLE = "CIRCLE"
@classmethod
- def from_str(cls, spatial: str) -> 'SpatialType':
+ def from_str(cls, spatial: str) -> "SpatialType":
try:
spatial = getattr(cls, spatial.upper())
except AttributeError:
diff --git a/python/sedona/core/formatMapper/__init__.py b/python/sedona/core/formatMapper/__init__.py
index 1292e27a9..0daadc217 100644
--- a/python/sedona/core/formatMapper/__init__.py
+++ b/python/sedona/core/formatMapper/__init__.py
@@ -19,4 +19,4 @@ from .geo_json_reader import GeoJsonReader
from .wkt_reader import WktReader
from .wkb_reader import WkbReader
-__all__ = ["GeoJsonReader", 'WktReader', 'WkbReader']
+__all__ = ["GeoJsonReader", "WktReader", "WkbReader"]
diff --git a/python/sedona/core/formatMapper/disc_utils.py b/python/sedona/core/formatMapper/disc_utils.py
index 2fd387203..34d6d1748 100644
--- a/python/sedona/core/formatMapper/disc_utils.py
+++ b/python/sedona/core/formatMapper/disc_utils.py
@@ -71,7 +71,9 @@ class LineStringRDDDiscLoader(DiscLoader):
def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
jvm = sc._jvm
line_string_rdd = LineStringRDD()
- srdd = SpatialObjectLoaderAdapter(jvm).load_line_string_spatial_rdd(sc._jsc, path)
+ srdd = SpatialObjectLoaderAdapter(jvm).load_line_string_spatial_rdd(
+ sc._jsc, path
+ )
line_string_rdd.set_srdd(srdd)
return line_string_rdd
@@ -99,7 +101,7 @@ loaders = {
GeoType.POINT: PointRDDDiscLoader,
GeoType.POLYGON: PolygonRDDDiscLoader,
GeoType.LINESTRING: LineStringRDDDiscLoader,
- GeoType.GEOMETRY: SpatialRDDDiscLoader
+ GeoType.GEOMETRY: SpatialRDDDiscLoader,
}
diff --git a/python/sedona/core/formatMapper/geo_json_reader.py b/python/sedona/core/formatMapper/geo_json_reader.py
index 8296b2b69..e0eeec695 100644
--- a/python/sedona/core/formatMapper/geo_json_reader.py
+++ b/python/sedona/core/formatMapper/geo_json_reader.py
@@ -32,17 +32,20 @@ class GeoJsonReader(GeoDataReader, metaclass=MultipleMeta):
:return: SpatialRDD
"""
jvm = sc._jvm
- srdd = jvm.GeoJsonReader.readToGeometryRDD(
- sc._jsc, inputPath
- )
+ srdd = jvm.GeoJsonReader.readToGeometryRDD(sc._jsc, inputPath)
spatial_rdd = SpatialRDD(sc)
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@classmethod
- def readToGeometryRDD(cls, sc: SparkContext, inputPath: str, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ sc: SparkContext,
+ inputPath: str,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param sc: SparkContext
@@ -53,7 +56,10 @@ class GeoJsonReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
srdd = jvm.GeoJsonReader.readToGeometryRDD(
- sc._jsc, inputPath, allowInvalidGeometries, skipSyntacticallyInvalidGeometries
+ sc._jsc,
+ inputPath,
+ allowInvalidGeometries,
+ skipSyntacticallyInvalidGeometries,
)
spatial_rdd = SpatialRDD(sc)
@@ -70,17 +76,19 @@ class GeoJsonReader(GeoDataReader, metaclass=MultipleMeta):
sc = rawTextRDD.ctx
jvm = sc._jvm
- srdd = jvm.GeoJsonReader.readToGeometryRDD(
- rawTextRDD._jrdd
- )
+ srdd = jvm.GeoJsonReader.readToGeometryRDD(rawTextRDD._jrdd)
spatial_rdd = SpatialRDD(sc)
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@classmethod
- def readToGeometryRDD(cls, rawTextRDD: RDD, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ rawTextRDD: RDD,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param rawTextRDD: RDD
diff --git a/python/sedona/core/formatMapper/geo_reader.py b/python/sedona/core/formatMapper/geo_reader.py
index d60b8053d..1cfb373a9 100644
--- a/python/sedona/core/formatMapper/geo_reader.py
+++ b/python/sedona/core/formatMapper/geo_reader.py
@@ -28,4 +28,5 @@ class GeoDataReader(metaclass=MultipleMeta):
@abc.abstractmethod
def readToGeometryRDD(cls, *args, **kwargs):
raise NotImplementedError(
- f"Instance of the class {cls.__class__.__name__} has to implement method readToGeometryRDD")
+ f"Instance of the class {cls.__class__.__name__} has to implement method readToGeometryRDD"
+ )
diff --git a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py b/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py
index 18d807b7a..48efe0eba 100644
--- a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py
+++ b/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py
@@ -37,10 +37,7 @@ class ShapefileReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
jsc = sc._jsc
- srdd = jvm.ShapefileReader.readToGeometryRDD(
- jsc,
- inputPath
- )
+ srdd = jvm.ShapefileReader.readToGeometryRDD(jsc, inputPath)
spatial_rdd = SpatialRDD(sc=sc)
spatial_rdd.set_srdd(srdd)
@@ -56,10 +53,7 @@ class ShapefileReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
jsc = sc._jsc
- srdd = jvm.ShapefileReader.readToPolygonRDD(
- jsc,
- inputPath
- )
+ srdd = jvm.ShapefileReader.readToPolygonRDD(jsc, inputPath)
spatial_rdd = PolygonRDD()
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@@ -74,10 +68,7 @@ class ShapefileReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
jsc = sc._jsc
- srdd = jvm.ShapefileReader.readToPointRDD(
- jsc,
- inputPath
- )
+ srdd = jvm.ShapefileReader.readToPointRDD(jsc, inputPath)
spatial_rdd = PointRDD()
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@@ -92,10 +83,7 @@ class ShapefileReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
jsc = sc._jsc
- srdd = jvm.ShapefileReader.readToLineStringRDD(
- jsc,
- inputPath
- )
+ srdd = jvm.ShapefileReader.readToLineStringRDD(jsc, inputPath)
spatial_rdd = LineStringRDD()
spatial_rdd.set_srdd(srdd)
return spatial_rdd
diff --git a/python/sedona/core/formatMapper/wkb_reader.py b/python/sedona/core/formatMapper/wkb_reader.py
index 3e9c9f67d..23cc27c9d 100644
--- a/python/sedona/core/formatMapper/wkb_reader.py
+++ b/python/sedona/core/formatMapper/wkb_reader.py
@@ -25,8 +25,14 @@ from sedona.utils.meta import MultipleMeta
class WkbReader(GeoDataReader, metaclass=MultipleMeta):
@classmethod
- def readToGeometryRDD(cls, sc: SparkContext, inputPath: str, wkbColumn: int, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ sc: SparkContext,
+ inputPath: str,
+ wkbColumn: int,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param sc:
@@ -38,14 +44,24 @@ class WkbReader(GeoDataReader, metaclass=MultipleMeta):
"""
jvm = sc._jvm
spatial_rdd = SpatialRDD(sc)
- srdd = jvm.WkbReader.readToGeometryRDD(sc._jsc, inputPath, wkbColumn, allowInvalidGeometries,
- skipSyntacticallyInvalidGeometries)
+ srdd = jvm.WkbReader.readToGeometryRDD(
+ sc._jsc,
+ inputPath,
+ wkbColumn,
+ allowInvalidGeometries,
+ skipSyntacticallyInvalidGeometries,
+ )
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@classmethod
- def readToGeometryRDD(cls, rawTextRDD: RDD, wkbColumn: int, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ rawTextRDD: RDD,
+ wkbColumn: int,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param rawTextRDD:
@@ -58,8 +74,12 @@ class WkbReader(GeoDataReader, metaclass=MultipleMeta):
jvm = sc._jvm
spatial_rdd = SpatialRDD(sc)
- srdd = jvm.WkbReader.readToGeometryRDD(rawTextRDD._jrdd, wkbColumn, allowInvalidGeometries,
- skipSyntacticallyInvalidGeometries)
+ srdd = jvm.WkbReader.readToGeometryRDD(
+ rawTextRDD._jrdd,
+ wkbColumn,
+ allowInvalidGeometries,
+ skipSyntacticallyInvalidGeometries,
+ )
spatial_rdd.set_srdd(srdd)
return spatial_rdd
diff --git a/python/sedona/core/formatMapper/wkt_reader.py b/python/sedona/core/formatMapper/wkt_reader.py
index e920ec682..499bbbda5 100644
--- a/python/sedona/core/formatMapper/wkt_reader.py
+++ b/python/sedona/core/formatMapper/wkt_reader.py
@@ -25,8 +25,14 @@ from sedona.utils.meta import MultipleMeta
class WktReader(GeoDataReader, metaclass=MultipleMeta):
@classmethod
- def readToGeometryRDD(cls, sc: SparkContext, inputPath: str, wktColumn: int, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ sc: SparkContext,
+ inputPath: str,
+ wktColumn: int,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param sc: SparkContext
@@ -37,16 +43,26 @@ class WktReader(GeoDataReader, metaclass=MultipleMeta):
:return:
"""
jvm = sc._jvm
- srdd = jvm.WktReader.readToGeometryRDD(sc._jsc, inputPath, wktColumn, allowInvalidGeometries,
- skipSyntacticallyInvalidGeometries)
+ srdd = jvm.WktReader.readToGeometryRDD(
+ sc._jsc,
+ inputPath,
+ wktColumn,
+ allowInvalidGeometries,
+ skipSyntacticallyInvalidGeometries,
+ )
spatial_rdd = SpatialRDD(sc)
spatial_rdd.set_srdd(srdd)
return spatial_rdd
@classmethod
- def readToGeometryRDD(cls, rawTextRDD: RDD, wktColumn: int, allowInvalidGeometries: bool,
- skipSyntacticallyInvalidGeometries: bool) -> SpatialRDD:
+ def readToGeometryRDD(
+ cls,
+ rawTextRDD: RDD,
+ wktColumn: int,
+ allowInvalidGeometries: bool,
+ skipSyntacticallyInvalidGeometries: bool,
+ ) -> SpatialRDD:
"""
:param rawTextRDD: RDD
@@ -58,7 +74,10 @@ class WktReader(GeoDataReader, metaclass=MultipleMeta):
sc = rawTextRDD.ctx
jvm = sc._jvm
srdd = jvm.WktReader.readToGeometryRDD(
- rawTextRDD._jrdd, wktColumn, allowInvalidGeometries, skipSyntacticallyInvalidGeometries
+ rawTextRDD._jrdd,
+ wktColumn,
+ allowInvalidGeometries,
+ skipSyntacticallyInvalidGeometries,
)
spatial_rdd = SpatialRDD(sc)
spatial_rdd.set_srdd(srdd)
diff --git a/python/sedona/core/geom/circle.py b/python/sedona/core/geom/circle.py
index ed091ef38..669259da5 100644
--- a/python/sedona/core/geom/circle.py
+++ b/python/sedona/core/geom/circle.py
@@ -18,7 +18,7 @@
import shapely
-if shapely.__version__.startswith('2.'):
+if shapely.__version__.startswith("2."):
from .shapely2.circle import Circle
else:
from .shapely1.circle import Circle
diff --git a/python/sedona/core/geom/envelope.py b/python/sedona/core/geom/envelope.py
index 402770394..db2ec5c4c 100644
--- a/python/sedona/core/geom/envelope.py
+++ b/python/sedona/core/geom/envelope.py
@@ -18,7 +18,7 @@
import shapely
-if shapely.__version__.startswith('2.'):
+if shapely.__version__.startswith("2."):
from .shapely2.envelope import Envelope
else:
from .shapely1.envelope import Envelope
diff --git a/python/sedona/core/geom/shapely1/circle.py b/python/sedona/core/geom/shapely1/circle.py
index d29b69255..787468f52 100644
--- a/python/sedona/core/geom/shapely1/circle.py
+++ b/python/sedona/core/geom/shapely1/circle.py
@@ -17,7 +17,14 @@
from math import sqrt
-from shapely.geometry import Polygon, Point, LineString, MultiPoint, MultiPolygon, MultiLineString
+from shapely.geometry import (
+ Polygon,
+ Point,
+ LineString,
+ MultiPoint,
+ MultiPolygon,
+ MultiLineString,
+)
from shapely.geometry.base import BaseGeometry
from sedona.core.geom.envelope import Envelope
@@ -32,19 +39,23 @@ class Circle(Polygon):
center_geometry_mbr = Envelope.from_shapely_geom(self.centerGeometry)
self.centerPoint = self.centerPoint = Point(
(center_geometry_mbr.minx + center_geometry_mbr.maxx) / 2.0,
- (center_geometry_mbr.miny + center_geometry_mbr.maxy) / 2.0
+ (center_geometry_mbr.miny + center_geometry_mbr.maxy) / 2.0,
)
width = center_geometry_mbr.maxx - center_geometry_mbr.minx
length = center_geometry_mbr.maxy - center_geometry_mbr.miny
- center_geometry_internal_radius = sqrt(width ** 2 + length ** 2) / 2.0
- self.radius = givenRadius if givenRadius > center_geometry_internal_radius else center_geometry_internal_radius
+ center_geometry_internal_radius = sqrt(width**2 + length**2) / 2.0
+ self.radius = (
+ givenRadius
+ if givenRadius > center_geometry_internal_radius
+ else center_geometry_internal_radius
+ )
self.MBR = Envelope(
self.centerPoint.x - self.radius,
self.centerPoint.x + self.radius,
self.centerPoint.y - self.radius,
- self.centerPoint.y + self.radius
+ self.centerPoint.y + self.radius,
)
super().__init__(self.centerPoint.buffer(self.radius))
@@ -61,13 +72,17 @@ class Circle(Polygon):
center_geometry_mbr = Envelope.from_shapely_geom(self.centerGeometry)
width = center_geometry_mbr.maxx - center_geometry_mbr.minx
length = center_geometry_mbr.maxy - center_geometry_mbr.miny
- center_geometry_internal_radius = sqrt(width ** 2 + length ** 2) / 2
- self.radius = givenRadius if givenRadius > center_geometry_internal_radius else center_geometry_internal_radius
+ center_geometry_internal_radius = sqrt(width**2 + length**2) / 2
+ self.radius = (
+ givenRadius
+ if givenRadius > center_geometry_internal_radius
+ else center_geometry_internal_radius
+ )
self.MBR = Envelope(
self.centerPoint.x - self.radius,
self.centerPoint.x + self.radius,
self.centerPoint.y - self.radius,
- self.centerPoint.y + self.radius
+ self.centerPoint.y + self.radius,
)
def covers(self, other: BaseGeometry) -> bool:
@@ -80,9 +95,13 @@ class Circle(Polygon):
elif isinstance(other, MultiPoint):
return all([self.covers_point(point) for point in other])
elif isinstance(other, MultiPolygon):
- return all([self.covers_linestring(polygon.exterior) for polygon in other.geoms])
+ return all(
+ [self.covers_linestring(polygon.exterior) for polygon in other.geoms]
+ )
elif isinstance(other, MultiLineString):
- return all([self.covers_linestring(linestring) for linestring in other.geoms])
+ return all(
+ [self.covers_linestring(linestring) for linestring in other.geoms]
+ )
else:
raise TypeError("Not supported")
@@ -114,7 +133,12 @@ class Circle(Polygon):
return self.MBR
def __str__(self):
- return "Circle of radius " + str(self.radius) + " around " + str(self.centerGeometry)
+ return (
+ "Circle of radius "
+ + str(self.radius)
+ + " around "
+ + str(self.centerGeometry)
+ )
@property
def __array_interface__(self):
diff --git a/python/sedona/core/geom/shapely1/envelope.py b/python/sedona/core/geom/shapely1/envelope.py
index a214c8277..bb16cbc30 100644
--- a/python/sedona/core/geom/shapely1/envelope.py
+++ b/python/sedona/core/geom/shapely1/envelope.py
@@ -22,6 +22,7 @@ from sedona.utils.decorators import require
import math
import pickle
+
class Envelope(Polygon):
def __init__(self, minx=0, maxx=1, miny=0, maxy=1):
@@ -29,27 +30,29 @@ class Envelope(Polygon):
self.maxx = maxx
self.miny = miny
self.maxy = maxy
- super().__init__([
- [self.minx, self.miny],
- [self.minx, self.maxy],
- [self.maxx, self.maxy],
- [self.maxx, self.miny]
- ])
+ super().__init__(
+ [
+ [self.minx, self.miny],
+ [self.minx, self.maxy],
+ [self.maxx, self.maxy],
+ [self.maxx, self.miny],
+ ]
+ )
def isClose(self, a, b) -> bool:
return math.isclose(a, b, rel_tol=1e-9)
def __eq__(self, other) -> bool:
- return self.isClose(self.minx, other.minx) and\
- self.isClose(self.miny, other.miny) and\
- self.isClose(self.maxx, other.maxx) and\
- self.isClose(self.maxy, other.maxy)
+ return (
+ self.isClose(self.minx, other.minx)
+ and self.isClose(self.miny, other.miny)
+ and self.isClose(self.maxx, other.maxx)
+ and self.isClose(self.maxy, other.maxy)
+ )
@require(["Envelope"])
def create_jvm_instance(self, jvm):
- return jvm.Envelope(
- self.minx, self.maxx, self.miny, self.maxy
- )
+ return jvm.Envelope(self.minx, self.maxx, self.miny, self.maxy)
@classmethod
def from_jvm_instance(cls, java_obj):
@@ -62,6 +65,7 @@ class Envelope(Polygon):
def to_bytes(self):
from sedona.utils.binary_parser import BinaryBuffer
+
bin_buffer = BinaryBuffer()
bin_buffer.put_double(self.minx)
bin_buffer.put_double(self.maxx)
@@ -83,13 +87,16 @@ class Envelope(Polygon):
return cls(min(x_coord), max(x_coord), min(y_coord), max(y_coord))
def __reduce__(self):
- return (self.__class__, (), dict(
- minx=self.minx,
- maxx=self.maxx,
- miny=self.miny,
- maxy=self.maxy,
-
- ))
+ return (
+ self.__class__,
+ (),
+ dict(
+ minx=self.minx,
+ maxx=self.maxx,
+ miny=self.miny,
+ maxy=self.maxy,
+ ),
+ )
def __getstate__(self):
return dict(
@@ -97,7 +104,6 @@ class Envelope(Polygon):
maxx=self.maxx,
miny=self.miny,
maxy=self.maxy,
-
)
def __setstate__(self, state):
diff --git a/python/sedona/core/geom/shapely2/circle.py b/python/sedona/core/geom/shapely2/circle.py
index 6b8b3a082..3147fc1f9 100644
--- a/python/sedona/core/geom/shapely2/circle.py
+++ b/python/sedona/core/geom/shapely2/circle.py
@@ -17,7 +17,15 @@
from math import sqrt
-from shapely.geometry import Polygon, Point, LineString, MultiPoint, MultiPolygon, MultiLineString, box
+from shapely.geometry import (
+ Polygon,
+ Point,
+ LineString,
+ MultiPoint,
+ MultiPolygon,
+ MultiLineString,
+ box,
+)
from shapely.geometry.base import BaseGeometry
from sedona.core.geom.envelope import Envelope
@@ -67,9 +75,13 @@ class Circle(Polygon):
elif isinstance(other, MultiPoint):
return all([self.covers_point(point) for point in other.geoms])
elif isinstance(other, MultiPolygon):
- return all([self.covers_linestring(polygon.exterior) for polygon in other.geoms])
+ return all(
+ [self.covers_linestring(polygon.exterior) for polygon in other.geoms]
+ )
elif isinstance(other, MultiLineString):
- return all([self.covers_linestring(linestring) for linestring in other.geoms])
+ return all(
+ [self.covers_linestring(linestring) for linestring in other.geoms]
+ )
else:
raise TypeError("Not supported")
@@ -103,4 +115,9 @@ class Circle(Polygon):
return Envelope(minx, maxx, miny, maxy)
def __str__(self):
- return "Circle of radius " + str(self.getRadius()) + " around " + str(self.centerGeometry)
+ return (
+ "Circle of radius "
+ + str(self.getRadius())
+ + " around "
+ + str(self.centerGeometry)
+ )
diff --git a/python/sedona/core/geom/shapely2/envelope.py b/python/sedona/core/geom/shapely2/envelope.py
index bf0c9dd30..fb9543416 100644
--- a/python/sedona/core/geom/shapely2/envelope.py
+++ b/python/sedona/core/geom/shapely2/envelope.py
@@ -54,10 +54,12 @@ class Envelope(Polygon):
def __eq__(self, other) -> bool:
minx, miny, maxx, maxy = self.bounds
other_minx, other_miny, other_maxx, other_maxy = other.bounds
- return self.isClose(minx, other_minx) and \
- self.isClose(miny, other_miny) and \
- self.isClose(maxx, other_maxx) and \
- self.isClose(maxy, other_maxy)
+ return (
+ self.isClose(minx, other_minx)
+ and self.isClose(miny, other_miny)
+ and self.isClose(maxx, other_maxx)
+ and self.isClose(maxy, other_maxy)
+ )
@require(["Envelope"])
def create_jvm_instance(self, jvm):
@@ -75,6 +77,7 @@ class Envelope(Polygon):
def to_bytes(self):
from sedona.utils.binary_parser import BinaryBuffer
+
minx, miny, maxx, maxy = self.bounds
bin_buffer = BinaryBuffer()
bin_buffer.put_double(minx)
@@ -108,6 +111,7 @@ class TmpEnvelopeForPickle:
immutable.
"""
+
def __init__(self, minx, maxx, miny, maxy):
self.minx = minx
self.maxx = maxx
@@ -115,11 +119,7 @@ class TmpEnvelopeForPickle:
self.maxy = maxy
def __getstate__(self):
- return dict(
- minx=self.minx,
- maxx=self.maxx,
- miny=self.miny,
- maxy=self.maxy)
+ return dict(minx=self.minx, maxx=self.maxx, miny=self.miny, maxy=self.maxy)
def __setstate__(self, state):
minx = state.get("minx", 0)
diff --git a/python/sedona/core/jvm/config.py b/python/sedona/core/jvm/config.py
index 96bc22ebe..d1fbc918d 100644
--- a/python/sedona/core/jvm/config.py
+++ b/python/sedona/core/jvm/config.py
@@ -27,7 +27,8 @@ import functools
import inspect
import warnings
-string_types = (type(b''), type(u''))
+string_types = (type(b""), type(""))
+
def is_greater_or_equal_version(version_a: str, version_b: str) -> bool:
if all([version_b, version_a]):
@@ -92,7 +93,7 @@ def deprecated(reason):
warnings.warn(
fmt1.format(name=func1.__name__, reason=reason),
category=DeprecationWarning,
- stacklevel=2
+ stacklevel=2,
)
return func1(*args, **kwargs)
@@ -122,7 +123,7 @@ def deprecated(reason):
warnings.warn(
fmt2.format(name=func2.__name__),
category=DeprecationWarning,
- stacklevel=2
+ stacklevel=2,
)
return func2(*args, **kwargs)
diff --git a/python/sedona/core/jvm/translate.py b/python/sedona/core/jvm/translate.py
index bddc5963d..d291f8d82 100644
--- a/python/sedona/core/jvm/translate.py
+++ b/python/sedona/core/jvm/translate.py
@@ -30,7 +30,9 @@ class JvmSedonaPythonConverter:
return self._jvm.PythonConverter.translateSpatialPairRDDToPython(spatial_rdd)
def translate_spatial_pair_rdd_with_list_to_python(self, spatial_rdd):
- return self._jvm.PythonConverter.translateSpatialPairRDDWithListToPython(spatial_rdd)
+ return self._jvm.PythonConverter.translateSpatialPairRDDWithListToPython(
+ spatial_rdd
+ )
def translate_python_rdd_to_java(self, java_rdd):
return self._jvm.PythonConverter.translatePythonRDDToJava(java_rdd)
@@ -103,13 +105,19 @@ class PythonRddToJavaRDDAdapter:
self._jvm = jvm
def deserialize_to_point_raw_rdd(self, java_spatial_rdd):
- return self._jvm.PythonRddToJavaRDDAdapter.deserializeToPointRawRDD(java_spatial_rdd)
+ return self._jvm.PythonRddToJavaRDDAdapter.deserializeToPointRawRDD(
+ java_spatial_rdd
+ )
def deserialize_to_polygon_raw_rdd(self, java_spatial_rdd):
- return self._jvm.PythonRddToJavaRDDAdapter.deserializeToPolygonRawRDD(java_spatial_rdd)
+ return self._jvm.PythonRddToJavaRDDAdapter.deserializeToPolygonRawRDD(
+ java_spatial_rdd
+ )
def deserialize_to_linestring_raw_rdd(self, java_spatial_rdd):
- return self._jvm.PythonRddToJavaRDDAdapter.deserializeToLineStringRawRDD(java_spatial_rdd)
+ return self._jvm.PythonRddToJavaRDDAdapter.deserializeToLineStringRawRDD(
+ java_spatial_rdd
+ )
class SpatialObjectLoaderAdapter:
diff --git a/python/sedona/core/spatialOperator/__init__.py b/python/sedona/core/spatialOperator/__init__.py
index 1022e395b..fc9e5f678 100644
--- a/python/sedona/core/spatialOperator/__init__.py
+++ b/python/sedona/core/spatialOperator/__init__.py
@@ -21,6 +21,4 @@ from .knn_query import KNNQuery
from .join_query_raw import JoinQueryRaw
from .range_query_raw import RangeQueryRaw
-__all__ = [
- "JoinQuery", "RangeQuery", "KNNQuery", "JoinQueryRaw", "RangeQueryRaw"
-]
+__all__ = ["JoinQuery", "RangeQuery", "KNNQuery", "JoinQueryRaw", "RangeQueryRaw"]
diff --git a/python/sedona/core/spatialOperator/join_params.py b/python/sedona/core/spatialOperator/join_params.py
index 232e6b279..c77224af7 100644
--- a/python/sedona/core/spatialOperator/join_params.py
+++ b/python/sedona/core/spatialOperator/join_params.py
@@ -30,7 +30,13 @@ class JoinParams:
joinBuildSide = attr.ib(type=str, default=JoinBuildSide.LEFT)
def jvm_instance(self, jvm):
- return JvmJoinParams(jvm, self.useIndex, self.considerBoundaryIntersection, self.indexType, self.joinBuildSide).jvm_instance
+ return JvmJoinParams(
+ jvm,
+ self.useIndex,
+ self.considerBoundaryIntersection,
+ self.indexType,
+ self.joinBuildSide,
+ ).jvm_instance
@attr.s
@@ -41,7 +47,12 @@ class JvmJoinParams(JvmObject):
joinBuildSide = attr.ib(type=str, default=JoinBuildSide.LEFT)
def _create_jvm_instance(self):
- return self.jvm_reference(self.useIndex, self.considerBoundaryIntersection, self.indexType.value, self.joinBuildSide)
+ return self.jvm_reference(
+ self.useIndex,
+ self.considerBoundaryIntersection,
+ self.indexType.value,
+ self.joinBuildSide,
+ )
@property
def jvm_reference(self):
diff --git a/python/sedona/core/spatialOperator/join_query.py b/python/sedona/core/spatialOperator/join_query.py
index d4483e7c2..d6d5f55bc 100644
--- a/python/sedona/core/spatialOperator/join_query.py
+++ b/python/sedona/core/spatialOperator/join_query.py
@@ -27,8 +27,13 @@ class JoinQuery:
@classmethod
@require(["JoinQuery"])
- def SpatialJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
- considerBoundaryIntersection: bool) -> RDD:
+ def SpatialJoinQuery(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> RDD:
"""
:param spatialRDD: SpatialRDD
@@ -38,13 +43,20 @@ class JoinQuery:
:return:
"""
- pair_rdd = JoinQueryRaw.SpatialJoinQuery(spatialRDD, queryRDD, useIndex, considerBoundaryIntersection)
+ pair_rdd = JoinQueryRaw.SpatialJoinQuery(
+ spatialRDD, queryRDD, useIndex, considerBoundaryIntersection
+ )
return pair_rdd.to_rdd()
@classmethod
@require(["JoinQuery"])
- def DistanceJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
- considerBoundaryIntersection: bool) -> RDD:
+ def DistanceJoinQuery(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> RDD:
"""
:param spatialRDD: SpatialRDD
@@ -54,12 +66,16 @@ class JoinQuery:
:return:
"""
- pair_rdd = JoinQueryRaw.DistanceJoinQuery(spatialRDD, queryRDD, useIndex, considerBoundaryIntersection)
+ pair_rdd = JoinQueryRaw.DistanceJoinQuery(
+ spatialRDD, queryRDD, useIndex, considerBoundaryIntersection
+ )
return pair_rdd.to_rdd()
@classmethod
@require(["JoinQuery"])
- def spatialJoin(cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams) -> RDD:
+ def spatialJoin(
+ cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams
+ ) -> RDD:
"""
:param queryWindowRDD: SpatialRDD
@@ -73,8 +89,13 @@ class JoinQuery:
@classmethod
@require(["JoinQuery"])
- def DistanceJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
- considerBoundaryIntersection: bool) -> RDD:
+ def DistanceJoinQueryFlat(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> RDD:
"""
:param spatialRDD: SpatialRDD
@@ -90,14 +111,20 @@ class JoinQuery:
:return:
"""
- pair_rdd = JoinQueryRaw.DistanceJoinQueryFlat(spatialRDD, queryRDD, useIndex,
- considerBoundaryIntersection)
+ pair_rdd = JoinQueryRaw.DistanceJoinQueryFlat(
+ spatialRDD, queryRDD, useIndex, considerBoundaryIntersection
+ )
return pair_rdd.to_rdd()
@classmethod
@require(["JoinQuery"])
- def SpatialJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
- considerBoundaryIntersection: bool) -> RDD:
+ def SpatialJoinQueryFlat(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> RDD:
"""
Function takes SpatialRDD and other SpatialRDD and based on two parameters
- useIndex
@@ -117,6 +144,7 @@ class JoinQuery:
[[GeoData(Polygon, ), GeoData()], [GeoData(), GeoData()], [GeoData(), GeoData()]]
"""
- pair_rdd = JoinQueryRaw.SpatialJoinQueryFlat(spatialRDD, queryRDD, useIndex,
- considerBoundaryIntersection)
+ pair_rdd = JoinQueryRaw.SpatialJoinQueryFlat(
+ spatialRDD, queryRDD, useIndex, considerBoundaryIntersection
+ )
return pair_rdd.to_rdd()
diff --git a/python/sedona/core/spatialOperator/join_query_raw.py b/python/sedona/core/spatialOperator/join_query_raw.py
index 4e1b54e21..d1b0f10a9 100644
--- a/python/sedona/core/spatialOperator/join_query_raw.py
+++ b/python/sedona/core/spatialOperator/join_query_raw.py
@@ -25,75 +25,90 @@ class JoinQueryRaw:
@classmethod
@require(["JoinQuery"])
- def SpatialJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:
+ def SpatialJoinQuery(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> SedonaPairRDDList:
jvm = spatialRDD._jvm
sc = spatialRDD._sc
srdd = jvm.JoinQuery.SpatialJoinQuery(
- spatialRDD._srdd,
- queryRDD._srdd,
- useIndex,
- considerBoundaryIntersection
+ spatialRDD._srdd, queryRDD._srdd, useIndex, considerBoundaryIntersection
)
return SedonaPairRDDList(srdd, sc)
@classmethod
@require(["JoinQuery"])
- def DistanceJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:
+ def DistanceJoinQuery(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> SedonaPairRDDList:
jvm = spatialRDD._jvm
sc = spatialRDD._sc
srdd = jvm.JoinQuery.DistanceJoinQuery(
- spatialRDD._srdd,
- queryRDD._srdd,
- useIndex,
- considerBoundaryIntersection
+ spatialRDD._srdd, queryRDD._srdd, useIndex, considerBoundaryIntersection
)
return SedonaPairRDDList(srdd, sc)
@classmethod
@require(["JoinQuery"])
- def spatialJoin(cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams) -> SedonaPairRDD:
+ def spatialJoin(
+ cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams
+ ) -> SedonaPairRDD:
jvm = queryWindowRDD._jvm
sc = queryWindowRDD._sc
jvm_join_params = joinParams.jvm_instance(jvm)
- srdd = jvm.JoinQuery.spatialJoin(queryWindowRDD._srdd, objectRDD._srdd, jvm_join_params)
+ srdd = jvm.JoinQuery.spatialJoin(
+ queryWindowRDD._srdd, objectRDD._srdd, jvm_join_params
+ )
return SedonaPairRDD(srdd, sc)
@classmethod
@require(["JoinQuery"])
- def DistanceJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDD:
+ def DistanceJoinQueryFlat(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> SedonaPairRDD:
jvm = spatialRDD._jvm
sc = spatialRDD._sc
spatial_join = jvm.JoinQuery.DistanceJoinQueryFlat
srdd = spatial_join(
- spatialRDD._srdd,
- queryRDD._srdd,
- useIndex,
- considerBoundaryIntersection
+ spatialRDD._srdd, queryRDD._srdd, useIndex, considerBoundaryIntersection
)
return SedonaPairRDD(srdd, sc)
@classmethod
@require(["JoinQuery"])
- def SpatialJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
- considerBoundaryIntersection: bool) -> SedonaPairRDD:
+ def SpatialJoinQueryFlat(
+ cls,
+ spatialRDD: SpatialRDD,
+ queryRDD: SpatialRDD,
+ useIndex: bool,
+ considerBoundaryIntersection: bool,
+ ) -> SedonaPairRDD:
jvm = spatialRDD._jvm
sc = spatialRDD._sc
spatial_join = jvm.JoinQuery.SpatialJoinQueryFlat
srdd = spatial_join(
- spatialRDD._srdd,
- queryRDD._srdd,
- useIndex,
- considerBoundaryIntersection
+ spatialRDD._srdd, queryRDD._srdd, useIndex, considerBoundaryIntersection
)
return SedonaPairRDD(srdd, sc)
diff --git a/python/sedona/core/spatialOperator/knn_query.py b/python/sedona/core/spatialOperator/knn_query.py
index 1d4c1224a..9e6905ce6 100644
--- a/python/sedona/core/spatialOperator/knn_query.py
+++ b/python/sedona/core/spatialOperator/knn_query.py
@@ -31,7 +31,13 @@ class KNNQuery:
@classmethod
@require(["KNNQuery", "GeometryAdapter"])
- def SpatialKnnQuery(self, spatialRDD: SpatialRDD, originalQueryPoint: BaseGeometry, k: int, useIndex: bool):
+ def SpatialKnnQuery(
+ self,
+ spatialRDD: SpatialRDD,
+ originalQueryPoint: BaseGeometry,
+ k: int,
+ useIndex: bool,
+ ):
"""
:param spatialRDD: spatialRDD
@@ -41,11 +47,17 @@ class KNNQuery:
:return: pyspark.RDD
"""
jvm = spatialRDD._jvm
- jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(jvm, originalQueryPoint)
+ jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(
+ jvm, originalQueryPoint
+ )
- knn_neighbours = jvm.KNNQuery.SpatialKnnQuery(spatialRDD._srdd, jvm_geom, k, useIndex)
+ knn_neighbours = jvm.KNNQuery.SpatialKnnQuery(
+ spatialRDD._srdd, jvm_geom, k, useIndex
+ )
- srdd = JvmSedonaPythonConverter(jvm).translate_geometry_seq_to_python(knn_neighbours)
+ srdd = JvmSedonaPythonConverter(jvm).translate_geometry_seq_to_python(
+ knn_neighbours
+ )
geoms_data = []
for arr in srdd:
diff --git a/python/sedona/core/spatialOperator/range_query.py b/python/sedona/core/spatialOperator/range_query.py
index df44ee374..b76ab8a77 100644
--- a/python/sedona/core/spatialOperator/range_query.py
+++ b/python/sedona/core/spatialOperator/range_query.py
@@ -26,8 +26,13 @@ class RangeQuery:
@classmethod
@require(["RangeQuery", "GeometryAdapter", "GeoSerializerData"])
- def SpatialRangeQuery(self, spatialRDD: SpatialRDD, rangeQueryWindow: BaseGeometry,
- considerBoundaryIntersection: bool, usingIndex: bool):
+ def SpatialRangeQuery(
+ self,
+ spatialRDD: SpatialRDD,
+ rangeQueryWindow: BaseGeometry,
+ considerBoundaryIntersection: bool,
+ usingIndex: bool,
+ ):
"""
:param spatialRDD:
@@ -36,5 +41,7 @@ class RangeQuery:
:param usingIndex:
:return:
"""
- j_srdd = RangeQueryRaw.SpatialRangeQuery(spatialRDD, rangeQueryWindow, considerBoundaryIntersection, usingIndex)
+ j_srdd = RangeQueryRaw.SpatialRangeQuery(
+ spatialRDD, rangeQueryWindow, considerBoundaryIntersection, usingIndex
+ )
return j_srdd.to_rdd()
diff --git a/python/sedona/core/spatialOperator/range_query_raw.py b/python/sedona/core/spatialOperator/range_query_raw.py
index dced4cd46..e20646ccb 100644
--- a/python/sedona/core/spatialOperator/range_query_raw.py
+++ b/python/sedona/core/spatialOperator/range_query_raw.py
@@ -27,8 +27,13 @@ class RangeQueryRaw:
@classmethod
@require(["RangeQuery", "GeometryAdapter", "GeoSerializerData"])
- def SpatialRangeQuery(self, spatialRDD: SpatialRDD, rangeQueryWindow: BaseGeometry,
- considerBoundaryIntersection: bool, usingIndex: bool) -> SedonaRDD:
+ def SpatialRangeQuery(
+ self,
+ spatialRDD: SpatialRDD,
+ rangeQueryWindow: BaseGeometry,
+ considerBoundaryIntersection: bool,
+ usingIndex: bool,
+ ) -> SedonaRDD:
"""
:param spatialRDD:
@@ -41,14 +46,12 @@ class RangeQueryRaw:
jvm = spatialRDD._jvm
sc = spatialRDD._sc
- jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(jvm, rangeQueryWindow)
+ jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(
+ jvm, rangeQueryWindow
+ )
- srdd = jvm. \
- RangeQuery.SpatialRangeQuery(
- spatialRDD._srdd,
- jvm_geom,
- considerBoundaryIntersection,
- usingIndex
+ srdd = jvm.RangeQuery.SpatialRangeQuery(
+ spatialRDD._srdd, jvm_geom, considerBoundaryIntersection, usingIndex
)
return SedonaRDD(srdd, sc)
diff --git a/python/sedona/core/spatialOperator/rdd.py b/python/sedona/core/spatialOperator/rdd.py
index 61969a12d..d96c65724 100644
--- a/python/sedona/core/spatialOperator/rdd.py
+++ b/python/sedona/core/spatialOperator/rdd.py
@@ -29,8 +29,9 @@ class SedonaRDD:
def to_rdd(self) -> RDD:
jvm = self.sc._jvm
- serialized = JvmSedonaPythonConverter(jvm). \
- translate_spatial_rdd_to_python(self.jsrdd)
+ serialized = JvmSedonaPythonConverter(jvm).translate_spatial_rdd_to_python(
+ self.jsrdd
+ )
return RDD(serialized, self.sc, SedonaPickler())
@@ -43,8 +44,9 @@ class SedonaPairRDD:
def to_rdd(self) -> RDD:
jvm = self.sc._jvm
- serialized = JvmSedonaPythonConverter(jvm). \
- translate_spatial_pair_rdd_to_python(self.jsrdd)
+ serialized = JvmSedonaPythonConverter(jvm).translate_spatial_pair_rdd_to_python(
+ self.jsrdd
+ )
return RDD(serialized, self.sc, SedonaPickler())
@@ -57,7 +59,8 @@ class SedonaPairRDDList:
def to_rdd(self):
jvm = self.sc._jvm
- serialized = JvmSedonaPythonConverter(jvm). \
- translate_spatial_pair_rdd_with_list_to_python(self.jsrdd)
+ serialized = JvmSedonaPythonConverter(
+ jvm
+ ).translate_spatial_pair_rdd_with_list_to_python(self.jsrdd)
return RDD(serialized, self.sc, SedonaPickler())
diff --git a/python/sedona/core/utils.py b/python/sedona/core/utils.py
index b9990ec38..f5ebada83 100644
--- a/python/sedona/core/utils.py
+++ b/python/sedona/core/utils.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+
class ImportedJvmLib:
_imported_libs = []
diff --git a/python/sedona/exceptions.py b/python/sedona/exceptions.py
index 18788608f..256c1ab9d 100644
--- a/python/sedona/exceptions.py
+++ b/python/sedona/exceptions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+
class InvalidParametersException(Exception):
"""
Exception added to handle invalid constructor parameters
diff --git a/python/sedona/maps/SedonaMapUtils.py b/python/sedona/maps/SedonaMapUtils.py
index 52cc47402..b4f68fb4c 100644
--- a/python/sedona/maps/SedonaMapUtils.py
+++ b/python/sedona/maps/SedonaMapUtils.py
@@ -34,7 +34,9 @@ class SedonaMapUtils:
if geometry_col is None:
geometry_col = SedonaMapUtils.__get_geometry_col__(df)
pandas_df = df.toPandas()
- if geometry_col is None: # No geometry column found even after searching schema, return Pandas Dataframe
+ if (
+ geometry_col is None
+ ): # No geometry column found even after searching schema, return Pandas Dataframe
return pandas_df
try:
import geopandas as gpd
@@ -73,14 +75,14 @@ class SedonaMapUtils:
geom_type = geom.geom_type
if geom_type not in type_list:
type_list.append(geom_type)
- if geom_type == 'Polygon':
+ if geom_type == "Polygon":
return geom.exterior.coords[0]
else:
return geom.coords[0]
@classmethod
def __extract_point_coordinate__(cls, geom):
- if geom.geom_type == 'Point':
+ if geom.geom_type == "Point":
return geom.coords[0]
@classmethod
@@ -91,5 +93,9 @@ class SedonaMapUtils:
@classmethod
def __is_geom_collection__(cls, geom_type):
- return geom_type == 'MultiPolygon' or geom_type == 'MultiLineString' or geom_type == 'MultiPoint' \
- or geom_type == 'GeometryCollection'
+ return (
+ geom_type == "MultiPolygon"
+ or geom_type == "MultiLineString"
+ or geom_type == "MultiPoint"
+ or geom_type == "GeometryCollection"
+ )
diff --git a/python/sedona/maps/SedonaPyDeck.py b/python/sedona/maps/SedonaPyDeck.py
index 7c2d8a763..367fffc9c 100644
--- a/python/sedona/maps/SedonaPyDeck.py
+++ b/python/sedona/maps/SedonaPyDeck.py
@@ -17,7 +17,15 @@
from types import ModuleType
-from pyspark.sql.types import FloatType, DoubleType, IntegerType, LongType, DecimalType, ShortType, ByteType
+from pyspark.sql.types import (
+ FloatType,
+ DoubleType,
+ IntegerType,
+ LongType,
+ DecimalType,
+ ShortType,
+ ByteType,
+)
from sedona.maps.SedonaMapUtils import SedonaMapUtils
@@ -26,8 +34,18 @@ class SedonaPyDeck:
# User Facing APIs
@classmethod
- def create_choropleth_map(cls, df, fill_color=None, plot_col=None, initial_view_state=None, map_style=None,
- map_provider=None, elevation_col=0, api_keys=None, stroked=True):
+ def create_choropleth_map(
+ cls,
+ df,
+ fill_color=None,
+ plot_col=None,
+ initial_view_state=None,
+ map_style=None,
+ map_provider=None,
+ elevation_col=0,
+ api_keys=None,
+ stroked=True,
+ ):
"""
Create a pydeck map with a choropleth layer added
:param elevation_col: Optional elevation for the polygons
@@ -43,17 +61,28 @@ class SedonaPyDeck:
"""
for field in df.schema.fields:
- if field.name == plot_col and field.dataType not in [IntegerType(), FloatType(), DoubleType(), LongType(),
- DecimalType(), ShortType(), ByteType()]:
- message = (f"'{field.name}' must be of numeric type. \nIf you are importing data from csv set "
- f"'inferSchema' option as true")
+ if field.name == plot_col and field.dataType not in [
+ IntegerType(),
+ FloatType(),
+ DoubleType(),
+ LongType(),
+ DecimalType(),
+ ShortType(),
+ ByteType(),
+ ]:
+ message = (
+ f"'{field.name}' must be of numeric type. \nIf you are importing data from csv set "
+ f"'inferSchema' option as true"
+ )
raise TypeError(message) from None
pdk = _try_import_pydeck()
if initial_view_state is None:
gdf = SedonaPyDeck._prepare_df_(df, add_coords=True)
- initial_view_state = pdk.data_utils.compute_view(gdf['coordinate_array_sedona'])
+ initial_view_state = pdk.data_utils.compute_view(
+ gdf["coordinate_array_sedona"]
+ )
else:
gdf = SedonaPyDeck._prepare_df_(df)
@@ -61,7 +90,7 @@ class SedonaPyDeck:
fill_color = SedonaPyDeck._create_default_fill_color_(gdf, plot_col)
choropleth_layer = pdk.Layer(
- 'GeoJsonLayer', # `type` positional argument is here
+ "GeoJsonLayer", # `type` positional argument is here
data=gdf,
auto_highlight=True,
get_fill_color=fill_color,
@@ -70,16 +99,30 @@ class SedonaPyDeck:
stroked=stroked,
extruded=True,
wireframe=True,
- pickable=True
+ pickable=True,
)
- return SedonaPyDeck._create_map_obj(layer=choropleth_layer, initial_view_state=initial_view_state, map_style=map_style,
- map_provider=map_provider, api_keys=api_keys)
+ return SedonaPyDeck._create_map_obj(
+ layer=choropleth_layer,
+ initial_view_state=initial_view_state,
+ map_style=map_style,
+ map_provider=map_provider,
+ api_keys=api_keys,
+ )
@classmethod
- def create_geometry_map(cls, df, fill_color="[85, 183, 177, 255]", line_color="[85, 183, 177, 255]",
- elevation_col=0, initial_view_state=None,
- map_style=None, map_provider=None, api_keys=None, stroked=True):
+ def create_geometry_map(
+ cls,
+ df,
+ fill_color="[85, 183, 177, 255]",
+ line_color="[85, 183, 177, 255]",
+ elevation_col=0,
+ initial_view_state=None,
+ map_style=None,
+ map_provider=None,
+ api_keys=None,
+ stroked=True,
+ ):
"""
Create a pydeck map with a GeoJsonLayer added for plotting given geometries.
:param line_color:
@@ -102,22 +145,44 @@ class SedonaPyDeck:
# if line_color == "[85, 183, 177, 255]":
# line_color = "[237, 119, 79]"
- layer = SedonaPyDeck._create_fat_layer_(gdf, fill_color=fill_color, elevation_col=elevation_col,
- line_color=line_color, stroked=stroked)
+ layer = SedonaPyDeck._create_fat_layer_(
+ gdf,
+ fill_color=fill_color,
+ elevation_col=elevation_col,
+ line_color=line_color,
+ stroked=stroked,
+ )
if initial_view_state is None:
- initial_view_state = pdk.data_utils.compute_view(gdf['coordinate_array_sedona'])
+ initial_view_state = pdk.data_utils.compute_view(
+ gdf["coordinate_array_sedona"]
+ )
if elevation_col != 0 and geom_type == "Polygon":
initial_view_state.pitch = 45 # If polygons are elevated, change the pitch to visualize the elevation better
- return SedonaPyDeck._create_map_obj(layer=layer, initial_view_state=initial_view_state, map_style=map_style,
- map_provider=map_provider, api_keys=api_keys)
+ return SedonaPyDeck._create_map_obj(
+ layer=layer,
+ initial_view_state=initial_view_state,
+ map_style=map_style,
+ map_provider=map_provider,
+ api_keys=api_keys,
+ )
@classmethod
- def create_scatterplot_map(cls, df, fill_color="[255, 140, 0]", radius_col=1, radius_min_pixels=1,
- radius_max_pixels=10, radius_scale=1, initial_view_state=None, map_style=None,
- map_provider=None, api_keys=None):
+ def create_scatterplot_map(
+ cls,
+ df,
+ fill_color="[255, 140, 0]",
+ radius_col=1,
+ radius_min_pixels=1,
+ radius_max_pixels=10,
+ radius_scale=1,
+ initial_view_state=None,
+ map_style=None,
+ map_provider=None,
+ api_keys=None,
+ ):
"""
Create a pydeck map with a scatterplot layer
:param radius_scale:
@@ -145,19 +210,35 @@ class SedonaPyDeck:
radius_min_pixels=radius_min_pixels,
radius_max_pixels=radius_max_pixels,
radius_scale=radius_scale,
- get_position='coordinate_array_sedona',
+ get_position="coordinate_array_sedona",
get_fill_color=fill_color,
)
if initial_view_state is None:
- initial_view_state = pdk.data_utils.compute_view(gdf['coordinate_array_sedona'])
-
- return SedonaPyDeck._create_map_obj(layer=layer, initial_view_state=initial_view_state, map_style=map_style,
- map_provider=map_provider, api_keys=api_keys)
+ initial_view_state = pdk.data_utils.compute_view(
+ gdf["coordinate_array_sedona"]
+ )
+
+ return SedonaPyDeck._create_map_obj(
+ layer=layer,
+ initial_view_state=initial_view_state,
+ map_style=map_style,
+ map_provider=map_provider,
+ api_keys=api_keys,
+ )
@classmethod
- def create_heatmap(cls, df, color_range=None, weight=1, aggregation="SUM", initial_view_state=None, map_style=None,
- map_provider=None, api_keys=None):
+ def create_heatmap(
+ cls,
+ df,
+ color_range=None,
+ weight=1,
+ aggregation="SUM",
+ initial_view_state=None,
+ map_style=None,
+ map_provider=None,
+ api_keys=None,
+ ):
"""
Create a pydeck map with a heatmap layer added
:param df: SedonaDataFrame to be used to plot the heatmap
@@ -181,7 +262,7 @@ class SedonaPyDeck:
[254, 178, 76],
[253, 141, 60],
[240, 59, 32],
- [240, 59, 32]
+ [240, 59, 32],
]
layer = pdk.Layer(
@@ -190,17 +271,24 @@ class SedonaPyDeck:
pickable=True,
opacity=0.8,
filled=True,
- get_position='coordinate_array_sedona',
+ get_position="coordinate_array_sedona",
aggregation=pdk.types.String(aggregation),
color_range=color_range,
- get_weight=weight
+ get_weight=weight,
)
if initial_view_state is None:
- initial_view_state = pdk.data_utils.compute_view(gdf['coordinate_array_sedona'])
-
- return SedonaPyDeck._create_map_obj(layer=layer, initial_view_state=initial_view_state, map_style=map_style,
- map_provider=map_provider, api_keys=api_keys)
+ initial_view_state = pdk.data_utils.compute_view(
+ gdf["coordinate_array_sedona"]
+ )
+
+ return SedonaPyDeck._create_map_obj(
+ layer=layer,
+ initial_view_state=initial_view_state,
+ map_style=map_style,
+ map_provider=map_provider,
+ api_keys=api_keys,
+ )
@classmethod
def _prepare_df_(cls, df, add_coords=False, geometry_col=None):
@@ -212,12 +300,13 @@ class SedonaPyDeck:
"""
if geometry_col is None:
geometry_col = SedonaMapUtils.__get_geometry_col__(df=df)
- gdf = SedonaMapUtils.__convert_to_gdf_or_pdf__(df, rename=False, geometry_col=geometry_col)
+ gdf = SedonaMapUtils.__convert_to_gdf_or_pdf__(
+ df, rename=False, geometry_col=geometry_col
+ )
if add_coords is True:
SedonaPyDeck._create_coord_column_(gdf=gdf, geometry_col=geometry_col)
return gdf
-
@classmethod
def _create_default_fill_color_(cls, gdf, plot_col):
"""
@@ -236,14 +325,18 @@ class SedonaPyDeck:
:param geometry_col: column with ST_Points
"""
type_list = []
- gdf['coordinate_array_sedona'] = gdf.apply(
- lambda val: list(SedonaMapUtils.__extract_coordinate__(val[geometry_col], type_list)), axis=1)
+ gdf["coordinate_array_sedona"] = gdf.apply(
+ lambda val: list(
+ SedonaMapUtils.__extract_coordinate__(val[geometry_col], type_list)
+ ),
+ axis=1,
+ )
@classmethod
def _create_fat_layer_(cls, gdf, fill_color, line_color, elevation_col, stroked):
pdk = _try_import_pydeck()
layer = pdk.Layer(
- 'GeoJsonLayer', # `type` positional argument is here
+ "GeoJsonLayer", # `type` positional argument is here
data=gdf,
auto_highlight=True,
get_fill_color=fill_color,
@@ -253,31 +346,38 @@ class SedonaPyDeck:
get_elevation=elevation_col,
get_line_color=line_color,
pickable=True,
- get_line_width=3
+ get_line_width=3,
)
return layer
# creates the final map object and handles the parameters for pydeck
@classmethod
- def _create_map_obj(cls, layer, initial_view_state, map_style, map_provider, api_keys):
+ def _create_map_obj(
+ cls, layer, initial_view_state, map_style, map_provider, api_keys
+ ):
pdk = _try_import_pydeck()
if map_provider is None:
- map_provider = 'carto'
+ map_provider = "carto"
if map_style is None:
- map_style = 'dark'
+ map_style = "dark"
# Default to mapbox if user selects 'satellite' map_style as pydeck uses mapbox and google maps for satellite basemap
- if map_style == 'satellite' and map_provider != 'google_maps':
- map_provider = 'mapbox'
+ if map_style == "satellite" and map_provider != "google_maps":
+ map_provider = "mapbox"
if api_keys is not None:
api_keys = api_keys
- return pdk.Deck(layers=[layer], initial_view_state=initial_view_state, map_style=map_style,
- map_provider=map_provider, api_keys=api_keys)
+ return pdk.Deck(
+ layers=[layer],
+ initial_view_state=initial_view_state,
+ map_style=map_style,
+ map_provider=map_provider,
+ api_keys=api_keys,
+ )
def _try_import_pydeck() -> ModuleType:
diff --git a/python/sedona/raster/awt_raster.py b/python/sedona/raster/awt_raster.py
index d95135943..e95935e3f 100644
--- a/python/sedona/raster/awt_raster.py
+++ b/python/sedona/raster/awt_raster.py
@@ -20,9 +20,8 @@ from .sample_model import SampleModel
class AWTRaster:
- """Raster data structure of Java AWT Raster used by GeoTools GridCoverage2D.
+ """Raster data structure of Java AWT Raster used by GeoTools GridCoverage2D."""
- """
min_x: int
min_y: int
width: int
@@ -30,7 +29,15 @@ class AWTRaster:
sample_model: SampleModel
data_buffer: DataBuffer
- def __init__(self, min_x, min_y, width, height, sample_model: SampleModel, data_buffer: DataBuffer):
+ def __init__(
+ self,
+ min_x,
+ min_y,
+ width,
+ height,
+ sample_model: SampleModel,
+ data_buffer: DataBuffer,
+ ):
if sample_model.width != width or sample_model.height != height:
raise RuntimeError("Size of the image does not match with the sample model")
self.min_x = min_x
diff --git a/python/sedona/raster/data_buffer.py b/python/sedona/raster/data_buffer.py
index 8826e26bd..28b28bbe3 100644
--- a/python/sedona/raster/data_buffer.py
+++ b/python/sedona/raster/data_buffer.py
@@ -32,7 +32,9 @@ class DataBuffer:
size: int
offsets: List[int]
- def __init__(self, data_type: int, bank_data: List[np.ndarray], size: int, offsets: List[int]):
+ def __init__(
+ self, data_type: int, bank_data: List[np.ndarray], size: int, offsets: List[int]
+ ):
self.data_type = data_type
self.bank_data = bank_data
self.size = size
diff --git a/python/sedona/raster/meta.py b/python/sedona/raster/meta.py
index b0013359d..a8ed687dc 100644
--- a/python/sedona/raster/meta.py
+++ b/python/sedona/raster/meta.py
@@ -26,6 +26,7 @@ class PixelAnchor(Enum):
transformation between these conventions.
"""
+
CENTER = 1
UPPER_LEFT = 2
@@ -39,7 +40,9 @@ class AffineTransform:
ip_y: float
pixel_anchor: PixelAnchor
- def __init__(self, scale_x, skew_y, skew_x, scale_y, ip_x, ip_y, pixel_anchor: PixelAnchor):
+ def __init__(
+ self, scale_x, skew_y, skew_x, scale_y, ip_x, ip_y, pixel_anchor: PixelAnchor
+ ):
self.scale_x = scale_x
self.skew_y = skew_y
self.skew_x = skew_x
@@ -56,8 +59,15 @@ class AffineTransform:
def translate(self, offset_x: float, offset_y: float):
new_ipx = self.ip_x + offset_x * self.scale_x + offset_y * self.skew_x
new_ipy = self.ip_y + offset_x * self.skew_y + offset_y * self.scale_y
- return AffineTransform(self.scale_x, self.skew_y, self.skew_x, self.scale_y,
- new_ipx, new_ipy, self.pixel_anchor)
+ return AffineTransform(
+ self.scale_x,
+ self.skew_y,
+ self.skew_x,
+ self.scale_y,
+ new_ipx,
+ new_ipy,
+ self.pixel_anchor,
+ )
def _do_change_pixel_anchor(self, from_anchor: PixelAnchor, to_anchor: PixelAnchor):
assert from_anchor != to_anchor
@@ -88,18 +98,21 @@ class AffineTransform:
new_m10 = old_m10 * m00 + old_m11 * m10
new_m11 = old_m10 * m01 + old_m11 * m11
new_m12 = old_m10 * m02 + old_m11 * m12 + old_m12
- return AffineTransform(new_m00, new_m10, new_m01, new_m11, new_m02, new_m12, to_anchor)
+ return AffineTransform(
+ new_m00, new_m10, new_m01, new_m11, new_m02, new_m12, to_anchor
+ )
def __repr__(self):
- return ("[ {} {} {}\n".format(self.scale_x, self.skew_x, self.ip_x) +
- " {} {} {}\n".format(self.skew_y, self.scale_y, self.ip_y) +
- " 0 0 1 ]")
+ return (
+ "[ {} {} {}\n".format(self.scale_x, self.skew_x, self.ip_x)
+ + " {} {} {}\n".format(self.skew_y, self.scale_y, self.ip_y)
+ + " 0 0 1 ]"
+ )
class SampleDimension:
- """Raster band metadata.
+ """Raster band metadata."""
- """
description: str
offset: float
scale: float
diff --git a/python/sedona/raster/raster_serde.py b/python/sedona/raster/raster_serde.py
index 63b740c5a..2bc396845 100644
--- a/python/sedona/raster/raster_serde.py
+++ b/python/sedona/raster/raster_serde.py
@@ -21,7 +21,13 @@ import struct
import zlib
import numpy as np
-from .sample_model import SampleModel, ComponentSampleModel, PixelInterleavedSampleModel, MultiPixelPackedSampleModel, SinglePixelPackedSampleModel
+from .sample_model import (
+ SampleModel,
+ ComponentSampleModel,
+ PixelInterleavedSampleModel,
+ MultiPixelPackedSampleModel,
+ SinglePixelPackedSampleModel,
+)
from .data_buffer import DataBuffer
from .awt_raster import AWTRaster
from .meta import AffineTransform, PixelAnchor, SampleDimension
@@ -52,7 +58,9 @@ def _deserialize(bio: BytesIO, raster_type: int) -> SedonaRaster:
if raster_type == RasterTypes.IN_DB:
# In-DB raster
awt_raster = _read_awt_raster(bio)
- return InDbSedonaRaster(width, height, bands_meta, affine_trans, crs_wkt, awt_raster)
+ return InDbSedonaRaster(
+ width, height, bands_meta, affine_trans, crs_wkt, awt_raster
+ )
else:
raise ValueError("unsupported raster_type: {}".format(raster_type))
@@ -63,19 +71,23 @@ def _read_grid_envelope(bio: BytesIO) -> Tuple[int, int, int, int]:
def _read_affine_transformation(bio: BytesIO) -> AffineTransform:
- scale_x, skew_y, skew_x, scale_y, ip_x, ip_y = struct.unpack("=dddddd", bio.read(8 * 6))
- return AffineTransform(scale_x, skew_y, skew_x, scale_y, ip_x, ip_y, PixelAnchor.CENTER)
+ scale_x, skew_y, skew_x, scale_y, ip_x, ip_y = struct.unpack(
+ "=dddddd", bio.read(8 * 6)
+ )
+ return AffineTransform(
+ scale_x, skew_y, skew_x, scale_y, ip_x, ip_y, PixelAnchor.CENTER
+ )
def _read_crs_wkt(bio: BytesIO) -> str:
- size, = struct.unpack("=i", bio.read(4))
+ (size,) = struct.unpack("=i", bio.read(4))
compressed_wkt = bio.read(size)
crs_wkt = zlib.decompress(compressed_wkt)
- return crs_wkt.decode('utf-8')
+ return crs_wkt.decode("utf-8")
def _read_sample_dimensions(bio: BytesIO) -> List[SampleDimension]:
- num_bands, = struct.unpack("=i", bio.read(4))
+ (num_bands,) = struct.unpack("=i", bio.read(4))
bands_meta = []
for i in range(num_bands):
description = _read_utf8_string(bio)
@@ -87,51 +99,76 @@ def _read_sample_dimensions(bio: BytesIO) -> List[SampleDimension]:
def _read_awt_raster(bio: BytesIO) -> AWTRaster:
min_x, min_y, width, height = struct.unpack("=iiii", bio.read(4 * 4))
- _ignore_java_object(bio) # image properties
- _ignore_java_object(bio) # color model
+ _ignore_java_object(bio) # image properties
+ _ignore_java_object(bio) # color model
min_x_1, min_y_1 = struct.unpack("=ii", bio.read(4 * 2))
if min_x_1 != min_x or min_y_1 != min_y:
- raise RuntimeError("malformed serialized raster: minx/miny of the image cannot match with minx/miny of the AWT raster")
+ raise RuntimeError(
+ "malformed serialized raster: minx/miny of the image cannot match with minx/miny of the AWT raster"
+ )
sample_model = _read_sample_model(bio)
data_buffer = _read_data_buffer(bio)
return AWTRaster(min_x, min_y, width, height, sample_model, data_buffer)
def _read_sample_model(bio: BytesIO) -> SampleModel:
- sample_model_type, data_type, width, height = struct.unpack("=iiii", bio.read(4 * 4))
+ sample_model_type, data_type, width, height = struct.unpack(
+ "=iiii", bio.read(4 * 4)
+ )
if sample_model_type == SampleModel.TYPE_BANDED:
bank_indices = _read_int_array(bio)
band_offsets = _read_int_array(bio)
- return ComponentSampleModel(data_type, width, height, 1, width, bank_indices, band_offsets)
+ return ComponentSampleModel(
+ data_type, width, height, 1, width, bank_indices, band_offsets
+ )
elif sample_model_type == SampleModel.TYPE_PIXEL_INTERLEAVED:
pixel_stride, scanline_stride = struct.unpack("=ii", bio.read(4 * 2))
band_offsets = _read_int_array(bio)
- return PixelInterleavedSampleModel(data_type, width, height, pixel_stride, scanline_stride, band_offsets)
- elif sample_model_type in [SampleModel.TYPE_COMPONENT, SampleModel.TYPE_COMPONENT_JAI]:
+ return PixelInterleavedSampleModel(
+ data_type, width, height, pixel_stride, scanline_stride, band_offsets
+ )
+ elif sample_model_type in [
+ SampleModel.TYPE_COMPONENT,
+ SampleModel.TYPE_COMPONENT_JAI,
+ ]:
pixel_stride, scanline_stride = struct.unpack("=ii", bio.read(4 * 2))
bank_indices = _read_int_array(bio)
band_offsets = _read_int_array(bio)
- return ComponentSampleModel(data_type, width, height, pixel_stride, scanline_stride, bank_indices, band_offsets)
+ return ComponentSampleModel(
+ data_type,
+ width,
+ height,
+ pixel_stride,
+ scanline_stride,
+ bank_indices,
+ band_offsets,
+ )
elif sample_model_type == SampleModel.TYPE_SINGLE_PIXEL_PACKED:
- scanline_stride, = struct.unpack("=i", bio.read(4))
+ (scanline_stride,) = struct.unpack("=i", bio.read(4))
bit_masks = _read_int_array(bio)
- return SinglePixelPackedSampleModel(data_type, width, height, scanline_stride, bit_masks)
+ return SinglePixelPackedSampleModel(
+ data_type, width, height, scanline_stride, bit_masks
+ )
elif sample_model_type == SampleModel.TYPE_MULTI_PIXEL_PACKED:
- num_bits, scanline_stride, data_bit_offset = struct.unpack("=iii", bio.read(4 * 3))
- return MultiPixelPackedSampleModel(data_type, width, height, num_bits, scanline_stride, data_bit_offset)
+ num_bits, scanline_stride, data_bit_offset = struct.unpack(
+ "=iii", bio.read(4 * 3)
+ )
+ return MultiPixelPackedSampleModel(
+ data_type, width, height, num_bits, scanline_stride, data_bit_offset
+ )
else:
raise RuntimeError(f"Unsupported SampleModel type: {sample_model_type}")
def _read_data_buffer(bio: BytesIO) -> DataBuffer:
- data_type, = struct.unpack("=i", bio.read(4))
+ (data_type,) = struct.unpack("=i", bio.read(4))
offsets = _read_int_array(bio)
- size, = struct.unpack("=i", bio.read(4))
+ (size,) = struct.unpack("=i", bio.read(4))
- num_banks, = struct.unpack("=i", bio.read(4))
+ (num_banks,) = struct.unpack("=i", bio.read(4))
banks = []
for i in range(num_banks):
- bank_size, = struct.unpack("=i", bio.read(4))
+ (bank_size,) = struct.unpack("=i", bio.read(4))
if data_type == DataBuffer.TYPE_BYTE:
np_array = np.frombuffer(bio.read(bank_size), dtype=np.uint8)
elif data_type == DataBuffer.TYPE_SHORT:
@@ -153,23 +190,23 @@ def _read_data_buffer(bio: BytesIO) -> DataBuffer:
def _read_utf8_string(bio: BytesIO) -> str:
- size, = struct.unpack("=i", bio.read(4))
+ (size,) = struct.unpack("=i", bio.read(4))
utf8_bytes = bio.read(size)
- return utf8_bytes.decode('utf-8')
+ return utf8_bytes.decode("utf-8")
def _ignore_java_object(bio: BytesIO):
- size, = struct.unpack("=i", bio.read(4))
+ (size,) = struct.unpack("=i", bio.read(4))
bio.read(size)
def _read_int_array(bio: BytesIO) -> List[int]:
- length, = struct.unpack("=i", bio.read(4))
+ (length,) = struct.unpack("=i", bio.read(4))
return [struct.unpack("=i", bio.read(4))[0] for _ in range(length)]
def _read_utf8_string_map(bio: BytesIO) -> Optional[Dict[str, str]]:
- size, = struct.unpack("=i", bio.read(4))
+ (size,) = struct.unpack("=i", bio.read(4))
if size == -1:
return None
params = {}
diff --git a/python/sedona/raster/sample_model.py b/python/sedona/raster/sample_model.py
index 4c5ac193e..2959b599f 100644
--- a/python/sedona/raster/sample_model.py
+++ b/python/sedona/raster/sample_model.py
@@ -27,6 +27,7 @@ class SampleModel(ABC):
SampleModel class in Java AWT.
"""
+
TYPE_BANDED = 1
TYPE_PIXEL_INTERLEAVED = 2
TYPE_SINGLE_PIXEL_PACKED = 3
@@ -47,7 +48,9 @@ class SampleModel(ABC):
@abstractmethod
def as_numpy(self, data_buffer: DataBuffer) -> np.ndarray:
- raise NotImplementedError("Abstract method as_numpy was not implemented by subclass")
+ raise NotImplementedError(
+ "Abstract method as_numpy was not implemented by subclass"
+ )
class ComponentSampleModel(SampleModel):
@@ -56,7 +59,16 @@ class ComponentSampleModel(SampleModel):
bank_indices: List[int]
band_offsets: List[int]
- def __init__(self, data_type, width, height, pixel_stride, scanline_stride, bank_indices, band_offsets):
+ def __init__(
+ self,
+ data_type,
+ width,
+ height,
+ pixel_stride,
+ scanline_stride,
+ bank_indices,
+ band_offsets,
+ ):
super().__init__(SampleModel.TYPE_COMPONENT, data_type, width, height)
self.pixel_stride = pixel_stride
self.scanline_stride = scanline_stride
@@ -71,7 +83,7 @@ class ComponentSampleModel(SampleModel):
bank_data = data_buffer.bank_data[bank_index]
offset = self.band_offsets[bank_index]
if offset != 0:
- bank_data = bank_data[offset:(offset + self.width * self.height)]
+ bank_data = bank_data[offset : (offset + self.width * self.height)]
band_arr = bank_data.reshape(self.height, self.width)
band_arrs.append(band_arr)
return np.array(band_arrs)
@@ -98,7 +110,9 @@ class PixelInterleavedSampleModel(SampleModel):
scanline_stride: int
band_offsets: List[int]
- def __init__(self, data_type, width, height, pixel_stride, scanline_stride, band_offsets):
+ def __init__(
+ self, data_type, width, height, pixel_stride, scanline_stride, band_offsets
+ ):
super().__init__(SampleModel.TYPE_PIXEL_INTERLEAVED, data_type, width, height)
self.pixel_stride = pixel_stride
self.scanline_stride = scanline_stride
@@ -107,9 +121,11 @@ class PixelInterleavedSampleModel(SampleModel):
def as_numpy(self, data_buffer: DataBuffer) -> np.ndarray:
num_bands = len(self.band_offsets)
bank_data = data_buffer.bank_data[0]
- if self.pixel_stride == num_bands and \
- self.scanline_stride == self.width * num_bands and \
- self.band_offsets == list(range(0, num_bands)):
+ if (
+ self.pixel_stride == num_bands
+ and self.scanline_stride == self.width * num_bands
+ and self.band_offsets == list(range(0, num_bands))
+ ):
# Fast path: no gapping in between band data, no band reordering
arr = bank_data.reshape(self.height, self.width, num_bands)
return np.transpose(arr, [2, 0, 1])
@@ -151,7 +167,9 @@ class SinglePixelPackedSampleModel(SampleModel):
for mask, bit_offset in zip(self.bit_masks, self.bit_offsets):
pixel.append((value & mask) >> bit_offset)
pixel_data.append(pixel)
- arr = np.array(pixel_data, dtype=bank_data.dtype).reshape(self.height, self.width, num_bands)
+ arr = np.array(pixel_data, dtype=bank_data.dtype).reshape(
+ self.height, self.width, num_bands
+ )
return np.transpose(arr, [2, 0, 1])
@@ -160,7 +178,9 @@ class MultiPixelPackedSampleModel(SampleModel):
scanline_stride: int
data_bit_offset: int
- def __init__(self, data_type, width, height, num_bits, scanline_stride, data_bit_offset):
+ def __init__(
+ self, data_type, width, height, num_bits, scanline_stride, data_bit_offset
+ ):
super().__init__(SampleModel.TYPE_MULTI_PIXEL_PACKED, data_type, width, height)
self.num_bits = num_bits
self.scanline_stride = scanline_stride
@@ -178,12 +198,12 @@ class MultiPixelPackedSampleModel(SampleModel):
pos = y * self.scanline_stride + self.data_bit_offset // bits_per_value
value = bank_data[pos]
shift = self.data_bit_offset % bits_per_value
- value = (value << shift)
+ value = value << shift
pixels: List[int] = []
while len(pixels) < self.width:
while shift < bits_per_value and len(pixels) < self.width:
pixels.append((value & mask) >> shift_right)
- value = (value << self.num_bits)
+ value = value << self.num_bits
shift += self.num_bits
pos += 1
value = bank_data[pos]
diff --git a/python/sedona/raster/sedona_raster.py b/python/sedona/raster/sedona_raster.py
index e5ecb3723..4520950ed 100644
--- a/python/sedona/raster/sedona_raster.py
+++ b/python/sedona/raster/sedona_raster.py
@@ -20,18 +20,18 @@ from abc import ABC, abstractmethod
from xml.etree.ElementTree import Element, SubElement, tostring
import numpy as np
-import rasterio # type: ignore
-import rasterio.env # type: ignore
-from rasterio.transform import Affine # type: ignore
-from rasterio.io import MemoryFile # type: ignore
-from rasterio.io import DatasetReader # type: ignore
+import rasterio # type: ignore
+import rasterio.env # type: ignore
+from rasterio.transform import Affine # type: ignore
+from rasterio.io import MemoryFile # type: ignore
+from rasterio.io import DatasetReader # type: ignore
try:
# for rasterio >= 1.3.0
- from rasterio._path import _parse_path as parse_path # type: ignore
+ from rasterio._path import _parse_path as parse_path # type: ignore
except:
# for rasterio >= 1.2.0
- from rasterio.path import parse_path # type: ignore
+ from rasterio.path import parse_path # type: ignore
from .awt_raster import AWTRaster
from .data_buffer import DataBuffer
@@ -53,39 +53,62 @@ def _rasterio_open(fp, driver=None):
return rasterio.open(fp, mode="r", driver=driver)
-def _generate_vrt_xml(src_path, data_type, width, height, geo_transform, crs_wkt, off_x, off_y, band_indices) -> bytes:
+def _generate_vrt_xml(
+ src_path,
+ data_type,
+ width,
+ height,
+ geo_transform,
+ crs_wkt,
+ off_x,
+ off_y,
+ band_indices,
+) -> bytes:
# Create root element
- root = Element('VRTDataset')
- root.set('rasterXSize', str(width))
- root.set('rasterYSize', str(height))
+ root = Element("VRTDataset")
+ root.set("rasterXSize", str(width))
+ root.set("rasterYSize", str(height))
# Add CRS
- if crs_wkt is not None and crs_wkt != '':
- srs = SubElement(root, 'SRS')
+ if crs_wkt is not None and crs_wkt != "":
+ srs = SubElement(root, "SRS")
srs.text = crs_wkt
# Add GeoTransform
- gt = SubElement(root, 'GeoTransform')
+ gt = SubElement(root, "GeoTransform")
gt.text = geo_transform
# Add bands
for i, band_index in enumerate(band_indices, start=1):
- band = SubElement(root, 'VRTRasterBand')
- band.set('dataType', data_type)
- band.set('band', str(i))
+ band = SubElement(root, "VRTRasterBand")
+ band.set("dataType", data_type)
+ band.set("band", str(i))
# Add source
- source = SubElement(band, 'SimpleSource')
- src_prop = SubElement(source, 'SourceFilename')
+ source = SubElement(band, "SimpleSource")
+ src_prop = SubElement(source, "SourceFilename")
src_prop.text = src_path
# Set source properties
- SubElement(source, 'SourceBand').text = str(band_index + 1)
- SubElement(source, 'SrcRect', {'xOff': str(off_x), 'yOff': str(off_y), 'xSize': str(width), 'ySize': str(height)})
- SubElement(source, 'DstRect', {'xOff': '0', 'yOff': '0', 'xSize': str(width), 'ySize': str(height)})
+ SubElement(source, "SourceBand").text = str(band_index + 1)
+ SubElement(
+ source,
+ "SrcRect",
+ {
+ "xOff": str(off_x),
+ "yOff": str(off_y),
+ "xSize": str(width),
+ "ySize": str(height),
+ },
+ )
+ SubElement(
+ source,
+ "DstRect",
+ {"xOff": "0", "yOff": "0", "xSize": str(width), "ySize": str(height)},
+ )
# Generate pretty XML
- xml_bytes = tostring(root, encoding='utf-8')
+ xml_bytes = tostring(root, encoding="utf-8")
return xml_bytes
@@ -96,8 +119,14 @@ class SedonaRaster(ABC):
_affine_trans: AffineTransform
_crs_wkt: str
- def __init__(self, width: int, height: int, bands_meta: List[SampleDimension],
- affine_trans: AffineTransform, crs_wkt: str):
+ def __init__(
+ self,
+ width: int,
+ height: int,
+ bands_meta: List[SampleDimension],
+ affine_trans: AffineTransform,
+ crs_wkt: str,
+ ):
self._width = width
self._height = height
self._bands_meta = bands_meta
@@ -131,9 +160,7 @@ class SedonaRaster(ABC):
@abstractmethod
def as_numpy(self) -> np.ndarray:
- """Get the bands data as an numpy array in CHW layout
-
- """
+ """Get the bands data as an numpy array in CHW layout"""
raise NotImplementedError()
def as_numpy_masked(self) -> np.ndarray:
@@ -150,9 +177,7 @@ class SedonaRaster(ABC):
@abstractmethod
def as_rasterio(self) -> DatasetReader:
- """Retrieve the raster as an rasterio DatasetReader
-
- """
+ """Retrieve the raster as an rasterio DatasetReader"""
raise NotImplementedError()
@abstractmethod
@@ -178,9 +203,15 @@ class InDbSedonaRaster(SedonaRaster):
rasterio_memfile: Optional[MemoryFile]
rasterio_dataset_reader: Optional[DatasetReader]
- def __init__(self, width: int, height: int, bands_meta: List[SampleDimension],
- affine_trans: AffineTransform, crs_wkt: str,
- awt_raster: AWTRaster):
+ def __init__(
+ self,
+ width: int,
+ height: int,
+ bands_meta: List[SampleDimension],
+ affine_trans: AffineTransform,
+ crs_wkt: str,
+ awt_raster: AWTRaster,
+ ):
super().__init__(width, height, bands_meta, affine_trans, crs_wkt)
self.awt_raster = awt_raster
self.rasterio_memfile = None
@@ -195,55 +226,72 @@ class InDbSedonaRaster(SedonaRaster):
return self.rasterio_dataset_reader
affine = Affine.from_gdal(
- self._affine_trans.ip_x, self._affine_trans.scale_x, self._affine_trans.skew_x,
- self._affine_trans.ip_y, self._affine_trans.skew_y, self._affine_trans.scale_y)
+ self._affine_trans.ip_x,
+ self._affine_trans.scale_x,
+ self._affine_trans.skew_x,
+ self._affine_trans.ip_y,
+ self._affine_trans.skew_y,
+ self._affine_trans.scale_y,
+ )
num_bands = len(self._bands_meta)
data_array = np.ascontiguousarray(self.as_numpy())
dtype = data_array.dtype
if dtype == np.uint8:
- data_type = 'Byte'
+ data_type = "Byte"
elif dtype == np.int8:
- data_type = 'Int8'
+ data_type = "Int8"
elif dtype == np.uint16:
- data_type = 'Uint16'
+ data_type = "Uint16"
elif dtype == np.int16:
- data_type = 'Int16'
+ data_type = "Int16"
elif dtype == np.uint32:
- data_type = 'UInt32'
+ data_type = "UInt32"
elif dtype == np.int32:
- data_type = 'Int32'
+ data_type = "Int32"
elif dtype == np.float32:
- data_type = 'Float32'
+ data_type = "Float32"
elif dtype == np.float64:
- data_type = 'Float64'
+ data_type = "Float64"
elif dtype == np.int64:
- data_type = 'Int64'
+ data_type = "Int64"
elif dtype == np.uint64:
- data_type = 'Uint64'
+ data_type = "Uint64"
else:
raise RuntimeError("unknown dtype: " + str(dtype))
arr_if = data_array.__array_interface__
- data_pointer = arr_if['data'][0]
- geotransform = (f"{self._affine_trans.ip_x}/{self._affine_trans.scale_x}/{self._affine_trans.skew_x}/" +
- f"{self._affine_trans.ip_y}/{self._affine_trans.skew_y}/{self._affine_trans.scale_y}")
+ data_pointer = arr_if["data"][0]
+ geotransform = (
+ f"{self._affine_trans.ip_x}/{self._affine_trans.scale_x}/{self._affine_trans.skew_x}/"
+ + f"{self._affine_trans.ip_y}/{self._affine_trans.skew_y}/{self._affine_trans.scale_y}"
+ )
# FIXME: GDAL 3.6 shipped with rasterio does not support
# SPATIALREFERENCE parameter, so we have to workaround this issue in a
# hacky way. If newer versions of rasterio bundle GDAL 3.7 then this
# won't be a problem. See https://gdal.org/drivers/raster/mem.html
- desc = (f"MEM:::DATAPOINTER={data_pointer},PIXELS={self._width},LINES={self._height},BANDS={num_bands}," +
- f"DATATYPE={data_type},GEOTRANSFORM={geotransform}")
+ desc = (
+ f"MEM:::DATAPOINTER={data_pointer},PIXELS={self._width},LINES={self._height},BANDS={num_bands},"
+ + f"DATATYPE={data_type},GEOTRANSFORM={geotransform}"
+ )
# construct a VRT to wrap this MEM dataset, with SRS set up properly
vrt_xml = _generate_vrt_xml(
- desc, data_type, self._width, self._height, geotransform.replace('/', ','), self._crs_wkt,
- 0, 0, list(range(num_bands)))
+ desc,
+ data_type,
+ self._width,
+ self._height,
+ geotransform.replace("/", ","),
+ self._crs_wkt,
+ 0,
+ 0,
+ list(range(num_bands)),
+ )
# dataset = _rasterio_open(desc, driver="MEM")
- self.rasterio_memfile = MemoryFile(vrt_xml, ext='.vrt')
- dataset = self.rasterio_memfile.open(driver='VRT')
+ self.rasterio_memfile = MemoryFile(vrt_xml, ext=".vrt")
+ dataset = self.rasterio_memfile.open(driver="VRT")
# XXX: dataset does not copy the data held by data_array, so we set
# data_array as a property of dataset to make sure that the lifetime of
@@ -254,8 +302,8 @@ class InDbSedonaRaster(SedonaRaster):
def close(self):
if self.rasterio_dataset_reader is not None:
- self.rasterio_dataset_reader.close()
- self.rasterio_dataset_reader = None
+ self.rasterio_dataset_reader.close()
+ self.rasterio_dataset_reader = None
if self.rasterio_memfile is not None:
self.rasterio_memfile.close()
self.rasterio_memfile = None
diff --git a/python/sedona/raster_utils/SedonaUtils.py b/python/sedona/raster_utils/SedonaUtils.py
index 4dbf9e399..ef0926ab6 100644
--- a/python/sedona/raster_utils/SedonaUtils.py
+++ b/python/sedona/raster_utils/SedonaUtils.py
@@ -15,8 +15,10 @@
# specific language governing permissions and limitations
# under the License.
+
class SedonaUtils:
@classmethod
def display_image(cls, df):
from IPython.display import display, HTML
+
display(HTML(df.toPandas().to_html(escape=False)))
diff --git a/python/sedona/register/geo_registrator.py b/python/sedona/register/geo_registrator.py
index 2df8474e6..8b80f3b17 100644
--- a/python/sedona/register/geo_registrator.py
+++ b/python/sedona/register/geo_registrator.py
@@ -50,11 +50,13 @@ class SedonaRegistrator:
def register(cls, spark: SparkSession):
return spark._jvm.SedonaSQLRegistrator.registerAll(spark._jsparkSession)
+
class PackageImporter:
@staticmethod
def import_jvm_lib(jvm) -> bool:
from sedona.core.utils import ImportedJvmLib
+
"""
Imports all the specified methods and functions in jvm
:param jvm: Jvm gateway from py4j
diff --git a/python/sedona/register/java_libs.py b/python/sedona/register/java_libs.py
index 6489cc4f3..931488d91 100644
--- a/python/sedona/register/java_libs.py
+++ b/python/sedona/register/java_libs.py
@@ -25,7 +25,9 @@ class SedonaJvmLib(Enum):
KNNQuery = "org.apache.sedona.core.spatialOperator.KNNQuery"
RangeQuery = "org.apache.sedona.core.spatialOperator.RangeQuery"
Envelope = "org.locationtech.jts.geom.Envelope"
- GeoSerializerData = "org.apache.sedona.python.wrapper.adapters.GeoSparkPythonConverter"
+ GeoSerializerData = (
+ "org.apache.sedona.python.wrapper.adapters.GeoSparkPythonConverter"
+ )
GeometryAdapter = "org.apache.sedona.python.wrapper.adapters.GeometryAdapter"
PointRDD = "org.apache.sedona.core.spatialRDD.PointRDD"
PolygonRDD = "org.apache.sedona.core.spatialRDD.PolygonRDD"
@@ -35,19 +37,27 @@ class SedonaJvmLib(Enum):
SpatialRDD = "org.apache.sedona.core.spatialRDD.SpatialRDD"
FileDataSplitter = "org.apache.sedona.common.enums.FileDataSplitter"
GeoJsonReader = "org.apache.sedona.core.formatMapper.GeoJsonReader"
- ShapeFileReader = "org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader"
+ ShapeFileReader = (
+ "org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader"
+ )
SedonaSQLRegistrator = "org.apache.sedona.sql.utils.SedonaSQLRegistrator"
StorageLevel = "org.apache.spark.storage.StorageLevel"
GridType = "org.apache.sedona.core.enums.GridType"
IndexType = "org.apache.sedona.core.enums.IndexType"
AdapterWrapper = "org.apache.sedona.python.wrapper.utils.PythonAdapterWrapper"
WktReader = "org.apache.sedona.core.formatMapper.WktReader"
- RawJvmIndexRDDSetter = "org.apache.sedona.python.wrapper.adapters.RawJvmIndexRDDSetter"
- SpatialObjectLoaderAdapter = "org.apache.sedona.python.wrapper.adapters.SpatialObjectLoaderAdapter"
+ RawJvmIndexRDDSetter = (
+ "org.apache.sedona.python.wrapper.adapters.RawJvmIndexRDDSetter"
+ )
+ SpatialObjectLoaderAdapter = (
+ "org.apache.sedona.python.wrapper.adapters.SpatialObjectLoaderAdapter"
+ )
WkbReader = "org.apache.sedona.core.formatMapper.WkbReader"
EnvelopeAdapter = "org.apache.sedona.python.wrapper.adapters.EnvelopeAdapter"
PythonConverter = "org.apache.sedona.python.wrapper.adapters.PythonConverter"
- PythonRddToJavaRDDAdapter = "org.apache.sedona.python.wrapper.adapters.PythonRddToJavaRDDAdapter"
+ PythonRddToJavaRDDAdapter = (
+ "org.apache.sedona.python.wrapper.adapters.PythonRddToJavaRDDAdapter"
+ )
st_constructors = "org.apache.spark.sql.sedona_sql.expressions.st_constructors"
st_functions = "org.apache.spark.sql.sedona_sql.expressions.st_functions"
st_predicates = "org.apache.spark.sql.sedona_sql.expressions.st_predicates"
@@ -55,7 +65,7 @@ class SedonaJvmLib(Enum):
SedonaContext = "org.apache.sedona.spark.SedonaContext"
@classmethod
- def from_str(cls, geo_lib: str) -> 'SedonaJvmLib':
+ def from_str(cls, geo_lib: str) -> "SedonaJvmLib":
try:
lib = getattr(cls, geo_lib.upper())
except AttributeError:
diff --git a/python/sedona/spark/SedonaContext.py b/python/sedona/spark/SedonaContext.py
index cda98a60f..5cba5df62 100644
--- a/python/sedona/spark/SedonaContext.py
+++ b/python/sedona/spark/SedonaContext.py
@@ -46,5 +46,6 @@ class SedonaContext:
This method is needed when the user wants to manually configure Sedona
:return: SparkSession.builder
"""
- return SparkSession.builder.config("spark.serializer", KryoSerializer.getName).\
- config("spark.kryo.registrator", SedonaKryoRegistrator.getName)
+ return SparkSession.builder.config(
+ "spark.serializer", KryoSerializer.getName
+ ).config("spark.kryo.registrator", SedonaKryoRegistrator.getName)
diff --git a/python/sedona/sql/__init__.py b/python/sedona/sql/__init__.py
index 5211b7fd2..c8e002fb0 100644
--- a/python/sedona/sql/__init__.py
+++ b/python/sedona/sql/__init__.py
@@ -30,9 +30,11 @@ from sedona.sql.st_functions import *
from sedona.sql.st_predicates import *
__all__ = (
- [name for name, obj in inspect.getmembers(sys.modules[__name__])] # get expected values from the modules
- + st_predicates.__all__
- + st_constructors.__all__
- + st_functions.__all__
- + st_aggregates.__all__
+ [
+ name for name, obj in inspect.getmembers(sys.modules[__name__])
+ ] # get expected values from the modules
+ + st_predicates.__all__
+ + st_constructors.__all__
+ + st_functions.__all__
+ + st_aggregates.__all__
)
diff --git a/python/sedona/sql/dataframe_api.py b/python/sedona/sql/dataframe_api.py
index aa78563a2..9d7c2f47b 100644
--- a/python/sedona/sql/dataframe_api.py
+++ b/python/sedona/sql/dataframe_api.py
@@ -48,13 +48,21 @@ def _convert_argument_to_java_column(arg: Any) -> Column:
return f.lit(arg)._jc
-def call_sedona_function(object_name: str, function_name: str, args: Union[Any, Tuple[Any]]) -> Column:
+def call_sedona_function(
+ object_name: str, function_name: str, args: Union[Any, Tuple[Any]]
+) -> Column:
spark = SparkSession.getActiveSession()
if spark is None:
- raise ValueError("No active spark session was detected. Unable to call sedona function.")
+ raise ValueError(
+ "No active spark session was detected. Unable to call sedona function."
+ )
# apparently a Column is an Iterable so we need to check for it explicitly
- if (not isinstance(args, Iterable)) or isinstance(args, str) or isinstance(args, Column):
+ if (
+ (not isinstance(args, Iterable))
+ or isinstance(args, str)
+ or isinstance(args, Column)
+ ):
args = [args]
args = map(_convert_argument_to_java_column, args)
@@ -78,7 +86,10 @@ def _get_type_list(annotated_type: Type) -> Tuple[Type, ...]:
"""
# in 3.8 there is a much nicer way to do this with typing.get_origin
# we have to be a bit messy until we drop support for 3.7
- if isinstance(annotated_type, typing._GenericAlias) and annotated_type.__origin__._name == "Union":
+ if (
+ isinstance(annotated_type, typing._GenericAlias)
+ and annotated_type.__origin__._name == "Union"
+ ):
# again, there is a really nice method for this in 3.8: typing.get_args
valid_types = annotated_type.__args__
else:
@@ -88,7 +99,7 @@ def _get_type_list(annotated_type: Type) -> Tuple[Type, ...]:
def _strip_extra_from_class_name(class_name):
- return class_name[len("<class '"):-len("'>")].split(".")[-1]
+ return class_name[len("<class '") : -len("'>")].split(".")[-1]
def _get_readable_name_for_type(type: Type) -> str:
@@ -118,7 +129,9 @@ def _get_bound_arguments(f: Callable, *args, **kwargs) -> Mapping[str, Any]:
return bound_args
-def _check_bound_arguments(bound_args: Mapping[str, Any], type_annotations: List[Type], function_name: str) -> None:
+def _check_bound_arguments(
+ bound_args: Mapping[str, Any], type_annotations: List[Type], function_name: str
+) -> None:
"""Check bound arguments against type annotations and raise a ValueError if any do not match.
:param bound_args: Bound arguments to check.
@@ -132,8 +145,12 @@ def _check_bound_arguments(bound_args: Mapping[str, Any], type_annotations: List
for bound_arg_name, bound_arg_value in bound_args.arguments.items():
annotated_type = type_annotations[bound_arg_name]
valid_type_list = _get_type_list(annotated_type)
- if not any([isinstance(bound_arg_value, valid_type) for valid_type in valid_type_list]):
- raise ValueError(f"Incorrect argument type: {bound_arg_name} for {function_name} should be {_get_readable_name_for_type(annotated_type)} but received {_strip_extra_from_class_name(str(type(bound_arg_value)))}.")
+ if not any(
+ [isinstance(bound_arg_value, valid_type) for valid_type in valid_type_list]
+ ):
+ raise ValueError(
+ f"Incorrect argument type: {bound_arg_name} for {function_name} should be {_get_readable_name_for_type(annotated_type)} but received {_strip_extra_from_class_name(str(type(bound_arg_value)))}."
+ )
def validate_argument_types(f: Callable) -> Callable:
@@ -146,12 +163,19 @@ def validate_argument_types(f: Callable) -> Callable:
:return: f wrapped with type validation checks.
:rtype: Callable
"""
+
def validated_function(*args, **kwargs) -> Column:
# all arguments are Columns or strings are always legal, so only check types when one of the arguments is not a column
- if not all([isinstance(x, Column) or isinstance(x, str) for x in itertools.chain(args, kwargs.values())]):
+ if not all(
+ [
+ isinstance(x, Column) or isinstance(x, str)
+ for x in itertools.chain(args, kwargs.values())
+ ]
+ ):
bound_args = _get_bound_arguments(f, *args, **kwargs)
type_annotations = typing.get_type_hints(f)
_check_bound_arguments(bound_args, type_annotations, f.__name__)
return f(*args, **kwargs)
+
return functools.update_wrapper(validated_function, f)
diff --git a/python/sedona/sql/exceptions.py b/python/sedona/sql/exceptions.py
index 3b7997066..6bbbe30a7 100644
--- a/python/sedona/sql/exceptions.py
+++ b/python/sedona/sql/exceptions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+
class GeometryUnavailableException(Exception):
def __init__(self, message):
diff --git a/python/sedona/sql/st_aggregates.py b/python/sedona/sql/st_aggregates.py
index 67315ecf8..5632594b4 100644
--- a/python/sedona/sql/st_aggregates.py
+++ b/python/sedona/sql/st_aggregates.py
@@ -21,7 +21,11 @@ from functools import partial
from pyspark.sql import Column
-from sedona.sql.dataframe_api import ColumnOrName, call_sedona_function, validate_argument_types
+from sedona.sql.dataframe_api import (
+ ColumnOrName,
+ call_sedona_function,
+ validate_argument_types,
+)
_call_aggregate_function = partial(call_sedona_function, "st_aggregates")
@@ -63,5 +67,8 @@ def ST_Union_Aggr(geometry: ColumnOrName) -> Column:
# Automatically populate __all__
-__all__ = [name for name, obj in inspect.getmembers(sys.modules[__name__])
- if inspect.isfunction(obj)]
+__all__ = [
+ name
+ for name, obj in inspect.getmembers(sys.modules[__name__])
+ if inspect.isfunction(obj)
+]
diff --git a/python/sedona/sql/st_constructors.py b/python/sedona/sql/st_constructors.py
index 594a36c66..b4e9aa686 100644
--- a/python/sedona/sql/st_constructors.py
+++ b/python/sedona/sql/st_constructors.py
@@ -22,14 +22,21 @@ from typing import Optional, Union
from pyspark.sql import Column
-from sedona.sql.dataframe_api import ColumnOrName, ColumnOrNameOrNumber, call_sedona_function, validate_argument_types
+from sedona.sql.dataframe_api import (
+ ColumnOrName,
+ ColumnOrNameOrNumber,
+ call_sedona_function,
+ validate_argument_types,
+)
_call_constructor_function = partial(call_sedona_function, "st_constructors")
@validate_argument_types
-def ST_GeomFromGeoHash(geohash: ColumnOrName, precision: Union[ColumnOrName, int]) -> Column:
+def ST_GeomFromGeoHash(
+ geohash: ColumnOrName, precision: Union[ColumnOrName, int]
+) -> Column:
"""Generate a geometry column from a geohash column at a specified precision.
:param geohash: Geohash string column to generate from.
@@ -41,8 +48,11 @@ def ST_GeomFromGeoHash(geohash: ColumnOrName, precision: Union[ColumnOrName, int
"""
return _call_constructor_function("ST_GeomFromGeoHash", (geohash, precision))
+
@validate_argument_types
-def ST_PointFromGeoHash(geohash: ColumnOrName, precision: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_PointFromGeoHash(
+ geohash: ColumnOrName, precision: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""Generate a point column from a geohash column at a specified precision.
:param geohash: Geohash string column to generate from.
@@ -94,7 +104,9 @@ def ST_GeomFromKML(kml_string: ColumnOrName) -> Column:
@validate_argument_types
-def ST_GeomFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_GeomFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a geometry column from a Well-Known Text (WKT) string column.
This is an alias of ST_GeomFromWKT.
@@ -107,8 +119,11 @@ def ST_GeomFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = No
return _call_constructor_function("ST_GeomFromText", args)
+
@validate_argument_types
-def ST_GeometryFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_GeometryFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a geometry column from a Well-Known Text (WKT) string column.
This is an alias of ST_GeomFromWKT.
@@ -133,6 +148,7 @@ def ST_GeomFromWKB(wkb: ColumnOrName) -> Column:
"""
return _call_constructor_function("ST_GeomFromWKB", wkb)
+
@validate_argument_types
def ST_GeomFromEWKB(wkb: ColumnOrName) -> Column:
"""Generate a geometry column from a Well-Known Binary (WKB) binary column.
@@ -146,7 +162,9 @@ def ST_GeomFromEWKB(wkb: ColumnOrName) -> Column:
@validate_argument_types
-def ST_GeomFromWKT(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_GeomFromWKT(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a geometry column from a Well-Known Text (WKT) string column.
This is an alias of ST_GeomFromText.
@@ -159,6 +177,7 @@ def ST_GeomFromWKT(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = Non
return _call_constructor_function("ST_GeomFromWKT", args)
+
@validate_argument_types
def ST_GeomFromEWKT(ewkt: ColumnOrName) -> Column:
"""Generate a geometry column from a OGC Extended Well-Known Text (WKT) string column.
@@ -211,8 +230,14 @@ def ST_Point(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber) -> Column:
"""
return _call_constructor_function("ST_Point", (x, y))
+
@validate_argument_types
-def ST_PointZ(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: ColumnOrNameOrNumber, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_PointZ(
+ x: ColumnOrNameOrNumber,
+ y: ColumnOrNameOrNumber,
+ z: ColumnOrNameOrNumber,
+ srid: Optional[ColumnOrNameOrNumber] = None,
+) -> Column:
"""Generates a 3D point geometry column from numeric values.
:param x: Either a number or numeric column representing the X coordinate of a point.
@@ -229,8 +254,14 @@ def ST_PointZ(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: ColumnOrNameO
args = (x, y, z) if srid is None else (x, y, z, srid)
return _call_constructor_function("ST_PointZ", args)
+
@validate_argument_types
-def ST_PointM(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, m: ColumnOrNameOrNumber, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_PointM(
+ x: ColumnOrNameOrNumber,
+ y: ColumnOrNameOrNumber,
+ m: ColumnOrNameOrNumber,
+ srid: Optional[ColumnOrNameOrNumber] = None,
+) -> Column:
"""Generates a 3D point geometry column from numeric values.
:param x: Either a number or numeric column representing the X coordinate of a point.
@@ -247,8 +278,15 @@ def ST_PointM(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, m: ColumnOrNameO
args = (x, y, m) if srid is None else (x, y, m, srid)
return _call_constructor_function("ST_PointM", args)
+
@validate_argument_types
-def ST_PointZM(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: ColumnOrNameOrNumber, m: ColumnOrNameOrNumber,srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_PointZM(
+ x: ColumnOrNameOrNumber,
+ y: ColumnOrNameOrNumber,
+ z: ColumnOrNameOrNumber,
+ m: ColumnOrNameOrNumber,
+ srid: Optional[ColumnOrNameOrNumber] = None,
+) -> Column:
"""Generates a 3D point geometry column from numeric values.
:param x: Either a number or numeric column representing the X coordinate of a point.
@@ -267,6 +305,7 @@ def ST_PointZM(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: ColumnOrName
args = (x, y, z, m) if srid is None else (x, y, z, m, srid)
return _call_constructor_function("ST_PointZM", args)
+
@validate_argument_types
def ST_PointFromText(coords: ColumnOrName, delimiter: ColumnOrName) -> Column:
"""Generate a point geometry column from coordinates separated by a delimiter and stored
@@ -281,8 +320,11 @@ def ST_PointFromText(coords: ColumnOrName, delimiter: ColumnOrName) -> Column:
"""
return _call_constructor_function("ST_PointFromText", (coords, delimiter))
+
@validate_argument_types
-def ST_PointFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_PointFromWKB(
+ wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a Point geometry column from a Well-Known Binary (WKB) binary column.
:param wkb: WKB binary column to generate from.
@@ -295,8 +337,11 @@ def ST_PointFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = No
args = (wkb) if srid is None else (wkb, srid)
return _call_constructor_function("ST_PointFromWKB", args)
+
@validate_argument_types
-def ST_LineFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_LineFromWKB(
+ wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a Line geometry column from a Well-Known Binary (WKB) binary column.
:param wkb: WKB binary column to generate from.
@@ -309,8 +354,11 @@ def ST_LineFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = Non
args = (wkb) if srid is None else (wkb, srid)
return _call_constructor_function("ST_LineFromWKB", args)
+
@validate_argument_types
-def ST_LinestringFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_LinestringFromWKB(
+ wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate a Line geometry column from a Well-Known Binary (WKB) binary column.
:param wkb: WKB binary column to generate from.
@@ -323,23 +371,32 @@ def ST_LinestringFromWKB(wkb: ColumnOrName, srid: Optional[ColumnOrNameOrNumber]
args = (wkb) if srid is None else (wkb, srid)
return _call_constructor_function("ST_LinestringFromWKB", args)
+
@validate_argument_types
-def ST_MakePointM(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, m: ColumnOrNameOrNumber) -> Column:
+def ST_MakePointM(
+ x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, m: ColumnOrNameOrNumber
+) -> Column:
"""Generate 3D M Point geometry.
- :param x: Either a number or numeric column representing the X coordinate of a point.
- :type x: ColumnOrNameOrNumber
- :param y: Either a number or numeric column representing the Y coordinate of a point.
- :type y: ColumnOrNameOrNumber
- :param m: Either a number or numeric column representing the M coordinate of a point
- :type m: ColumnOrNameOrNumber
- :return: Point geometry column generated from the coordinate values.
- :rtype: Column
- """
+ :param x: Either a number or numeric column representing the X coordinate of a point.
+ :type x: ColumnOrNameOrNumber
+ :param y: Either a number or numeric column representing the Y coordinate of a point.
+ :type y: ColumnOrNameOrNumber
+ :param m: Either a number or numeric column representing the M coordinate of a point
+ :type m: ColumnOrNameOrNumber
+ :return: Point geometry column generated from the coordinate values.
+ :rtype: Column
+ """
return _call_constructor_function("ST_MakePointM", (x, y, m))
+
@validate_argument_types
-def ST_MakePoint(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: Optional[ColumnOrNameOrNumber] = None, m: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_MakePoint(
+ x: ColumnOrNameOrNumber,
+ y: ColumnOrNameOrNumber,
+ z: Optional[ColumnOrNameOrNumber] = None,
+ m: Optional[ColumnOrNameOrNumber] = None,
+) -> Column:
"""Generate a 2D, 3D Z or 4D ZM Point geometry. If z is None then a 2D point is generated.
This function doesn't support M coordinates for creating a 4D ZM Point in Dataframe API.
@@ -361,8 +418,15 @@ def ST_MakePoint(x: ColumnOrNameOrNumber, y: ColumnOrNameOrNumber, z: Optional[C
args = args + (m,)
return _call_constructor_function("ST_MakePoint", (args))
+
@validate_argument_types
-def ST_MakeEnvelope(min_x: ColumnOrNameOrNumber, min_y: ColumnOrNameOrNumber, max_x: ColumnOrNameOrNumber, max_y: ColumnOrNameOrNumber, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_MakeEnvelope(
+ min_x: ColumnOrNameOrNumber,
+ min_y: ColumnOrNameOrNumber,
+ max_x: ColumnOrNameOrNumber,
+ max_y: ColumnOrNameOrNumber,
+ srid: Optional[ColumnOrNameOrNumber] = None,
+) -> Column:
"""Generate a polygon geometry column from the minimum and maximum coordinates of an envelope with an option to add SRID
:param min_x: Minimum X coordinate for the envelope.
@@ -384,8 +448,14 @@ def ST_MakeEnvelope(min_x: ColumnOrNameOrNumber, min_y: ColumnOrNameOrNumber, ma
return _call_constructor_function("ST_MakeEnvelope", args)
+
@validate_argument_types
-def ST_PolygonFromEnvelope(min_x: ColumnOrNameOrNumber, min_y: ColumnOrNameOrNumber, max_x: ColumnOrNameOrNumber, max_y: ColumnOrNameOrNumber) -> Column:
+def ST_PolygonFromEnvelope(
+ min_x: ColumnOrNameOrNumber,
+ min_y: ColumnOrNameOrNumber,
+ max_x: ColumnOrNameOrNumber,
+ max_y: ColumnOrNameOrNumber,
+) -> Column:
"""Generate a polygon geometry column from the minimum and maximum coordinates of an envelope.
:param min_x: Minimum X coordinate for the envelope.
@@ -399,7 +469,9 @@ def ST_PolygonFromEnvelope(min_x: ColumnOrNameOrNumber, min_y: ColumnOrNameOrNum
:return: Polygon geometry column representing the envelope described by the coordinate bounds.
:rtype: Column
"""
- return _call_constructor_function("ST_PolygonFromEnvelope", (min_x, min_y, max_x, max_y))
+ return _call_constructor_function(
+ "ST_PolygonFromEnvelope", (min_x, min_y, max_x, max_y)
+ )
@validate_argument_types
@@ -416,8 +488,11 @@ def ST_PolygonFromText(coords: ColumnOrName, delimiter: ColumnOrName) -> Column:
"""
return _call_constructor_function("ST_PolygonFromText", (coords, delimiter))
+
@validate_argument_types
-def ST_MPolyFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_MPolyFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate multiPolygon geometry from a multiPolygon WKT representation.
:param wkt: multiPolygon WKT string column to generate from.
@@ -429,8 +504,11 @@ def ST_MPolyFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = N
return _call_constructor_function("ST_MPolyFromText", args)
+
@validate_argument_types
-def ST_MLineFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_MLineFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate multiLineString geometry from a multiLineString WKT representation.
:param wkt: multiLineString WKT string column to generate from.
@@ -442,8 +520,11 @@ def ST_MLineFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = N
return _call_constructor_function("ST_MLineFromText", args)
+
@validate_argument_types
-def ST_MPointFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_MPointFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate MultiPoint geometry from a MultiPoint WKT representation.
:param wkt: MultiPoint WKT string column to generate from.
@@ -457,8 +538,11 @@ def ST_MPointFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] =
return _call_constructor_function("ST_MPointFromText", args)
+
@validate_argument_types
-def ST_GeomCollFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None) -> Column:
+def ST_GeomCollFromText(
+ wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber] = None
+) -> Column:
"""Generate GeometryCollection geometry from a GeometryCollection WKT representation.
:param wkt: GeometryCollection WKT string column to generate from.
@@ -474,5 +558,8 @@ def ST_GeomCollFromText(wkt: ColumnOrName, srid: Optional[ColumnOrNameOrNumber]
# Automatically populate __all__
-__all__ = [name for name, obj in inspect.getmembers(sys.modules[__name__])
- if inspect.isfunction(obj)]
+__all__ = [
+ name
+ for name, obj in inspect.getmembers(sys.modules[__name__])
+ if inspect.isfunction(obj)
+]
diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py
index fccc1c565..05d90c2e8 100644
--- a/python/sedona/sql/st_functions.py
+++ b/python/sedona/sql/st_functions.py
@@ -22,11 +22,17 @@ from typing import Optional, Union
from pyspark.sql import Column
-from sedona.sql.dataframe_api import call_sedona_function, ColumnOrName, ColumnOrNameOrNumber, validate_argument_types
+from sedona.sql.dataframe_api import (
+ call_sedona_function,
+ ColumnOrName,
+ ColumnOrNameOrNumber,
+ validate_argument_types,
+)
_call_st_function = partial(call_sedona_function, "st_functions")
+
@validate_argument_types
def GeometryType(geometry: ColumnOrName):
"""Return the type of the geometry as a string.
@@ -39,6 +45,7 @@ def GeometryType(geometry: ColumnOrName):
"""
return _call_st_function("GeometryType", geometry)
+
@validate_argument_types
def ST_3DDistance(a: ColumnOrName, b: ColumnOrName) -> Column:
"""Calculate the 3-dimensional minimum Cartesian distance between two geometry columns.
@@ -52,8 +59,13 @@ def ST_3DDistance(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_st_function("ST_3DDistance", (a, b))
+
@validate_argument_types
-def ST_AddMeasure(geom: ColumnOrName, measureStart: Union[ColumnOrName, float], measureEnd: Union[ColumnOrName, float]) -> Column:
+def ST_AddMeasure(
+ geom: ColumnOrName,
+ measureStart: Union[ColumnOrName, float],
+ measureEnd: Union[ColumnOrName, float],
+) -> Column:
"""Interpolate measure values with the provided start and end points and return the result geometry.
:param geom: Geometry column to use in the calculation.
@@ -67,8 +79,13 @@ def ST_AddMeasure(geom: ColumnOrName, measureStart: Union[ColumnOrName, float],
"""
return _call_st_function("ST_AddMeasure", (geom, measureStart, measureEnd))
+
@validate_argument_types
-def ST_AddPoint(line_string: ColumnOrName, point: ColumnOrName, index: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_AddPoint(
+ line_string: ColumnOrName,
+ point: ColumnOrName,
+ index: Optional[Union[ColumnOrName, int]] = None,
+) -> Column:
"""Add a point to either the end of a linestring or a specified index.
If index is not provided then point will be added to the end of line_string.
@@ -96,6 +113,7 @@ def ST_Area(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Area", geometry)
+
@validate_argument_types
def ST_AreaSpheroid(geometry: ColumnOrName) -> Column:
"""Calculate the area of a geometry using WGS84 spheroid.
@@ -132,8 +150,11 @@ def ST_AsEWKB(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_AsEWKB", geometry)
+
@validate_argument_types
-def ST_AsHEXEWKB(geometry: ColumnOrName, endian: Optional[ColumnOrName] = None) -> Column:
+def ST_AsHEXEWKB(
+ geometry: ColumnOrName, endian: Optional[ColumnOrName] = None
+) -> Column:
"""Generate the Extended Well-Known Binary representation of a geometry as Hex string.
:param geometry: Geometry to generate EWKB for.
@@ -160,7 +181,9 @@ def ST_AsEWKT(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_AsGeoJSON(geometry: ColumnOrName, type: Optional[Union[ColumnOrName, str]] = None) -> Column:
+def ST_AsGeoJSON(
+ geometry: ColumnOrName, type: Optional[Union[ColumnOrName, str]] = None
+) -> Column:
"""Generate the GeoJSON style representation of a geometry column.
:param geometry: Geometry column to generate GeoJSON for.
@@ -234,6 +257,7 @@ def ST_BestSRID(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_BestSRID", geometry)
+
@validate_argument_types
def ST_ShiftLongitude(geometry: ColumnOrName) -> Column:
"""Shifts longitudes between -180..0 degrees to 180..360 degrees and vice versa.
@@ -245,6 +269,7 @@ def ST_ShiftLongitude(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ShiftLongitude", geometry)
+
@validate_argument_types
def ST_Boundary(geometry: ColumnOrName) -> Column:
"""Calculate the closure of the combinatorial boundary of a geometry column.
@@ -258,7 +283,12 @@ def ST_Boundary(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_Buffer(geometry: ColumnOrName, buffer: ColumnOrNameOrNumber, useSpheroid: Optional[Union[ColumnOrName, bool]] = None, parameters: Optional[Union[ColumnOrName, str]] = None) -> Column:
+def ST_Buffer(
+ geometry: ColumnOrName,
+ buffer: ColumnOrNameOrNumber,
+ useSpheroid: Optional[Union[ColumnOrName, bool]] = None,
+ parameters: Optional[Union[ColumnOrName, str]] = None,
+) -> Column:
"""Calculate a geometry that represents all points whose distance from the
input geometry column is equal to or less than a given amount.
@@ -320,7 +350,9 @@ def ST_Collect(*geometries: ColumnOrName) -> Column:
@validate_argument_types
-def ST_CollectionExtract(collection: ColumnOrName, geom_type: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_CollectionExtract(
+ collection: ColumnOrName, geom_type: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""Extract a specific type of geometry from a geometry collection column
as a multi-geometry column.
@@ -351,7 +383,11 @@ def ST_ClosestPoint(a: ColumnOrName, b: ColumnOrName) -> Column:
@validate_argument_types
-def ST_ConcaveHull(geometry: ColumnOrName, pctConvex: Union[ColumnOrName, float], allowHoles: Optional[Union[ColumnOrName, bool]] = None) -> Column:
+def ST_ConcaveHull(
+ geometry: ColumnOrName,
+ pctConvex: Union[ColumnOrName, float],
+ allowHoles: Optional[Union[ColumnOrName, bool]] = None,
+) -> Column:
"""Generate the cancave hull of a geometry column.
:param geometry: Geometry column to generate a cancave hull for.
@@ -363,9 +399,14 @@ def ST_ConcaveHull(geometry: ColumnOrName, pctConvex: Union[ColumnOrName, float]
:return: Concave hull of geometry as a geometry column.
:rtype: Column
"""
- args = (geometry, pctConvex) if allowHoles is None else (geometry, pctConvex, allowHoles)
+ args = (
+ (geometry, pctConvex)
+ if allowHoles is None
+ else (geometry, pctConvex, allowHoles)
+ )
return _call_st_function("ST_ConcaveHull", args)
+
@validate_argument_types
def ST_ConvexHull(geometry: ColumnOrName) -> Column:
"""Generate the convex hull of a geometry column.
@@ -377,6 +418,7 @@ def ST_ConvexHull(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ConvexHull", geometry)
+
@validate_argument_types
def ST_CrossesDateLine(a: ColumnOrName) -> Column:
"""Check whether geometry a crosses the International Date Line.
@@ -388,6 +430,7 @@ def ST_CrossesDateLine(a: ColumnOrName) -> Column:
"""
return _call_st_function("ST_CrossesDateLine", (a))
+
@validate_argument_types
def ST_Dimension(geometry: ColumnOrName):
"""Calculate the inherent dimension of a geometry column.
@@ -399,6 +442,7 @@ def ST_Dimension(geometry: ColumnOrName):
"""
return _call_st_function("ST_Dimension", geometry)
+
@validate_argument_types
def ST_Difference(a: ColumnOrName, b: ColumnOrName) -> Column:
"""Calculate the difference of two geometry columns. This difference
@@ -428,6 +472,7 @@ def ST_Distance(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Distance", (a, b))
+
@validate_argument_types
def ST_DistanceSpheroid(a: ColumnOrName, b: ColumnOrName) -> Column:
"""Calculate the geodesic distance between two geometry columns using WGS84 spheroid.
@@ -441,8 +486,13 @@ def ST_DistanceSpheroid(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_st_function("ST_DistanceSpheroid", (a, b))
+
@validate_argument_types
-def ST_DistanceSphere(a: ColumnOrName, b: ColumnOrName, radius: Optional[Union[ColumnOrName, float]] = 6371008.0) -> Column:
+def ST_DistanceSphere(
+ a: ColumnOrName,
+ b: ColumnOrName,
+ radius: Optional[Union[ColumnOrName, float]] = 6371008.0,
+) -> Column:
"""Calculate the haversine/great-circle distance between two geometry columns using a given radius.
:param a: Geometry column to use in the calculation.
@@ -456,6 +506,7 @@ def ST_DistanceSphere(a: ColumnOrName, b: ColumnOrName, radius: Optional[Union[C
"""
return _call_st_function("ST_DistanceSphere", (a, b, radius))
+
@validate_argument_types
def ST_Dump(geometry: ColumnOrName) -> Column:
"""Returns an array of geometries that are members of a multi-geometry
@@ -508,7 +559,12 @@ def ST_Envelope(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_Expand(geometry: ColumnOrName, deltaX_uniformDelta: Union[ColumnOrName, float], deltaY: Optional[Union[ColumnOrName, float]] = None, deltaZ: Optional[Union[ColumnOrName, float]] = None) -> Column:
+def ST_Expand(
+ geometry: ColumnOrName,
+ deltaX_uniformDelta: Union[ColumnOrName, float],
+ deltaY: Optional[Union[ColumnOrName, float]] = None,
+ deltaZ: Optional[Union[ColumnOrName, float]] = None,
+) -> Column:
"""Expand the given geometry column by a constant unit in each direction
:param geometry: Geometry column to calculate the envelope of.
@@ -570,7 +626,11 @@ def ST_Force_2D(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_GeneratePoints(geometry: ColumnOrName, numPoints: Union[ColumnOrName, int], seed:Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_GeneratePoints(
+ geometry: ColumnOrName,
+ numPoints: Union[ColumnOrName, int],
+ seed: Optional[Union[ColumnOrName, int]] = None,
+) -> Column:
"""Generate random points in given geometry.
:param geometry: Geometry column to hash.
@@ -587,6 +647,7 @@ def ST_GeneratePoints(geometry: ColumnOrName, numPoints: Union[ColumnOrName, int
return _call_st_function("ST_GeneratePoints", args)
+
@validate_argument_types
def ST_GeoHash(geometry: ColumnOrName, precision: Union[ColumnOrName, int]) -> Column:
"""Return the geohash of a geometry column at a given precision level.
@@ -600,10 +661,14 @@ def ST_GeoHash(geometry: ColumnOrName, precision: Union[ColumnOrName, int]) -> C
"""
return _call_st_function("ST_GeoHash", (geometry, precision))
+
@validate_argument_types
-def ST_GeometricMedian(geometry: ColumnOrName, tolerance: Optional[Union[ColumnOrName, float]] = 1e-6,
- max_iter: Optional[Union[ColumnOrName, int]] = 1000,
- fail_if_not_converged: Optional[Union[ColumnOrName, bool]] = False) -> Column:
+def ST_GeometricMedian(
+ geometry: ColumnOrName,
+ tolerance: Optional[Union[ColumnOrName, float]] = 1e-6,
+ max_iter: Optional[Union[ColumnOrName, int]] = 1000,
+ fail_if_not_converged: Optional[Union[ColumnOrName, bool]] = False,
+) -> Column:
"""Computes the approximate geometric median of a MultiPoint geometry using the Weiszfeld algorithm.
The geometric median provides a centrality measure that is less sensitive to outlier points than the centroid.
The algorithm will iterate until the distance change between successive iterations is less than the
@@ -656,7 +721,9 @@ def ST_GeometryType(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_H3CellDistance(cell1: Union[ColumnOrName, int], cell2: Union[ColumnOrName, int]) -> Column:
+def ST_H3CellDistance(
+ cell1: Union[ColumnOrName, int], cell2: Union[ColumnOrName, int]
+) -> Column:
"""Cover Geometry with H3 Cells and return a List of Long type cell IDs
:param cell: start cell
:type cell: long
@@ -670,7 +737,11 @@ def ST_H3CellDistance(cell1: Union[ColumnOrName, int], cell2: Union[ColumnOrName
@validate_argument_types
-def ST_H3CellIDs(geometry: ColumnOrName, level: Union[ColumnOrName, int], full_cover: Union[ColumnOrName, bool]) -> Column:
+def ST_H3CellIDs(
+ geometry: ColumnOrName,
+ level: Union[ColumnOrName, int],
+ full_cover: Union[ColumnOrName, bool],
+) -> Column:
"""Cover Geometry with H3 Cells and return a List of Long type cell IDs
:param geometry: Geometry column to generate cell IDs
:type geometry: ColumnOrName
@@ -686,7 +757,11 @@ def ST_H3CellIDs(geometry: ColumnOrName, level: Union[ColumnOrName, int], full_c
@validate_argument_types
-def ST_H3KRing(cell: Union[ColumnOrName, int], k: Union[ColumnOrName, int], exact_ring: Union[ColumnOrName, bool]) -> Column:
+def ST_H3KRing(
+ cell: Union[ColumnOrName, int],
+ k: Union[ColumnOrName, int],
+ exact_ring: Union[ColumnOrName, bool],
+) -> Column:
"""Cover Geometry with H3 Cells and return a List of Long type cell IDs
:param cell: original cell
:type cell: long
@@ -804,7 +879,9 @@ def ST_IsSimple(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_IsValid(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_IsValid(
+ geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""Check if a geometry is well formed.
:param geometry: Geometry column to check in.
@@ -818,8 +895,11 @@ def ST_IsValid(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]]
args = (geometry,) if flag is None else (geometry, flag)
return _call_st_function("ST_IsValid", args)
+
@validate_argument_types
-def ST_IsValidDetail(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_IsValidDetail(
+ geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""
Return a row of valid, reason and location. valid defines the validity of geometry, reason defines the
reason why it is not valid and location defines the location where it is not valid
@@ -835,6 +915,7 @@ def ST_IsValidDetail(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName,
args = (geometry,) if flag is None else (geometry, flag)
return _call_st_function("ST_IsValidDetail", args)
+
@validate_argument_types
def ST_IsValidTrajectory(geometry: ColumnOrName) -> Column:
"""
@@ -848,8 +929,11 @@ def ST_IsValidTrajectory(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_IsValidTrajectory", (geometry))
+
@validate_argument_types
-def ST_IsValidReason(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_IsValidReason(
+ geometry: ColumnOrName, flag: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""
Provides a text description of why a geometry is not valid or states that it is valid.
An optional flag parameter can be provided for additional options.
@@ -864,6 +948,7 @@ def ST_IsValidReason(geometry: ColumnOrName, flag: Optional[Union[ColumnOrName,
args = (geometry,) if flag is None else (geometry, flag)
return _call_st_function("ST_IsValidReason", args)
+
@validate_argument_types
def ST_Length(geometry: ColumnOrName) -> Column:
"""Calculate the length of a linestring geometry.
@@ -875,6 +960,7 @@ def ST_Length(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Length", geometry)
+
@validate_argument_types
def ST_Length2D(geometry: ColumnOrName) -> Column:
"""Calculate the length of a linestring geometry.
@@ -886,6 +972,7 @@ def ST_Length2D(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Length2D", geometry)
+
@validate_argument_types
def ST_LengthSpheroid(geometry: ColumnOrName) -> Column:
"""Calculate the perimeter of a geometry using WGS84 spheroid.
@@ -897,6 +984,7 @@ def ST_LengthSpheroid(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_LengthSpheroid", geometry)
+
@validate_argument_types
def ST_LineFromMultiPoint(geometry: ColumnOrName) -> Column:
"""Creates a LineString from a MultiPoint geometry.
@@ -910,7 +998,9 @@ def ST_LineFromMultiPoint(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_LineInterpolatePoint(geometry: ColumnOrName, fraction: ColumnOrNameOrNumber) -> Column:
+def ST_LineInterpolatePoint(
+ geometry: ColumnOrName, fraction: ColumnOrNameOrNumber
+) -> Column:
"""Calculate a point that is interpolated along a linestring.
:param geometry: Linestring geometry column to interpolate from.
@@ -922,6 +1012,7 @@ def ST_LineInterpolatePoint(geometry: ColumnOrName, fraction: ColumnOrNameOrNumb
"""
return _call_st_function("ST_LineInterpolatePoint", (geometry, fraction))
+
@validate_argument_types
def ST_LineLocatePoint(linestring: ColumnOrName, point: ColumnOrName) -> Column:
"""Returns a double between 0 and 1 representing the location of the closest point on a LineString to the given Point, as a fraction of 2d line length.
@@ -950,7 +1041,11 @@ def ST_LineMerge(multi_line_string: ColumnOrName) -> Column:
@validate_argument_types
-def ST_LineSubstring(line_string: ColumnOrName, start_fraction: ColumnOrNameOrNumber, end_fraction: ColumnOrNameOrNumber) -> Column:
+def ST_LineSubstring(
+ line_string: ColumnOrName,
+ start_fraction: ColumnOrNameOrNumber,
+ end_fraction: ColumnOrNameOrNumber,
+) -> Column:
"""Generate a substring of a linestring geometry column.
:param line_string: Linestring geometry column to generate from.
@@ -962,10 +1057,17 @@ def ST_LineSubstring(line_string: ColumnOrName, start_fraction: ColumnOrNameOrNu
:return: Smaller linestring that runs from start_fraction to end_fraction of line_string as a linestring geometry column, will be null if either start_fraction or end_fraction are outside the interval [0, 1].
:rtype: Column
"""
- return _call_st_function("ST_LineSubstring", (line_string, start_fraction, end_fraction))
+ return _call_st_function(
+ "ST_LineSubstring", (line_string, start_fraction, end_fraction)
+ )
+
@validate_argument_types
-def ST_LocateAlong(geom: ColumnOrName, measure: Union[ColumnOrName, float], offset: Optional[Union[ColumnOrName, float]] = None) -> Column:
+def ST_LocateAlong(
+ geom: ColumnOrName,
+ measure: Union[ColumnOrName, float],
+ offset: Optional[Union[ColumnOrName, float]] = None,
+) -> Column:
"""return locations along a measure geometry that have the given measure value.
:param geom:
@@ -980,6 +1082,7 @@ def ST_LocateAlong(geom: ColumnOrName, measure: Union[ColumnOrName, float], offs
args = (geom, measure) if offset is None else (geom, measure, offset)
return _call_st_function("ST_LocateAlong", args)
+
@validate_argument_types
def ST_LongestLine(geom1: ColumnOrName, geom2: ColumnOrName) -> Column:
"""Compute the longest line between the two geometries
@@ -993,6 +1096,7 @@ def ST_LongestLine(geom1: ColumnOrName, geom2: ColumnOrName) -> Column:
"""
return _call_st_function("ST_LongestLine", (geom1, geom2))
+
@validate_argument_types
def ST_HasZ(geom: ColumnOrName) -> Column:
"""Check whether geometry has Z coordinate
@@ -1004,6 +1108,7 @@ def ST_HasZ(geom: ColumnOrName) -> Column:
"""
return _call_st_function("ST_HasZ", geom)
+
@validate_argument_types
def ST_HasM(geom: ColumnOrName) -> Column:
"""Check whether geometry has M coordinate
@@ -1027,6 +1132,7 @@ def ST_M(geom: ColumnOrName) -> Column:
"""
return _call_st_function("ST_M", geom)
+
@validate_argument_types
def ST_MMin(geom: ColumnOrName) -> Column:
"""Return the minimum M coordinate of a geometry.
@@ -1038,6 +1144,7 @@ def ST_MMin(geom: ColumnOrName) -> Column:
"""
return _call_st_function("ST_MMin", geom)
+
@validate_argument_types
def ST_MMax(geom: ColumnOrName) -> Column:
"""Return the maximum M coordinate of a geometry.
@@ -1064,6 +1171,7 @@ def ST_MakeLine(geom1: ColumnOrName, geom2: Optional[ColumnOrName] = None) -> Co
args = (geom1,) if geom2 is None else (geom1, geom2)
return _call_st_function("ST_MakeLine", args)
+
@validate_argument_types
def ST_Points(geometry: ColumnOrName) -> Column:
"""Creates a MultiPoint geometry consisting of all the coordinates of the input geometry
@@ -1075,6 +1183,7 @@ def ST_Points(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Points", (geometry))
+
@validate_argument_types
def ST_Polygon(line_string: ColumnOrName, srid: ColumnOrNameOrNumber) -> Column:
"""Create a polygon built from the given LineString and sets the spatial reference system from the srid.
@@ -1088,6 +1197,7 @@ def ST_Polygon(line_string: ColumnOrName, srid: ColumnOrNameOrNumber) -> Column:
"""
return _call_st_function("ST_Polygon", (line_string, srid))
+
@validate_argument_types
def ST_Polygonize(geometry: ColumnOrName) -> Column:
"""Generates a GeometryCollection composed of polygons that are formed from the linework of a set of input geometries.
@@ -1099,8 +1209,11 @@ def ST_Polygonize(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Polygonize", (geometry))
+
@validate_argument_types
-def ST_MakePolygon(line_string: ColumnOrName, holes: Optional[ColumnOrName] = None) -> Column:
+def ST_MakePolygon(
+ line_string: ColumnOrName, holes: Optional[ColumnOrName] = None
+) -> Column:
"""Create a polygon geometry from a linestring describing the exterior ring as well as an array of linestrings describing holes.
:param line_string: Closed linestring geometry column that describes the exterior ring of the polygon.
@@ -1115,7 +1228,9 @@ def ST_MakePolygon(line_string: ColumnOrName, holes: Optional[ColumnOrName] = No
@validate_argument_types
-def ST_MakeValid(geometry: ColumnOrName, keep_collapsed: Optional[Union[ColumnOrName, bool]] = None) -> Column:
+def ST_MakeValid(
+ geometry: ColumnOrName, keep_collapsed: Optional[Union[ColumnOrName, bool]] = None
+) -> Column:
"""Convert an invalid geometry in a geometry column into a valid geometry.
:param geometry: Geometry column that contains the invalid geometry.
@@ -1133,13 +1248,14 @@ def ST_MakeValid(geometry: ColumnOrName, keep_collapsed: Optional[Union[ColumnOr
def ST_MaximumInscribedCircle(geometry: ColumnOrName) -> Column:
"""Finds the largest circle that is contained within a geometry, or which does not overlap any lines and points
- :param geometry:
- :type geometry: ColumnOrName
- :return: Row of center point, nearest point and radius
- :rtype: Column
- """
+ :param geometry:
+ :type geometry: ColumnOrName
+ :return: Row of center point, nearest point and radius
+ :rtype: Column
+ """
return _call_st_function("ST_MaximumInscribedCircle", geometry)
+
@validate_argument_types
def ST_MaxDistance(geom1: ColumnOrName, geom2: ColumnOrName) -> Column:
"""Calculate the maximum distance between two furthest points in the geometries
@@ -1153,6 +1269,7 @@ def ST_MaxDistance(geom1: ColumnOrName, geom2: ColumnOrName) -> Column:
"""
return _call_st_function("ST_MaxDistance", (geom1, geom2))
+
@validate_argument_types
def ST_MinimumClearance(geometry: ColumnOrName) -> Column:
"""Calculate the minimum clearance between two vertices
@@ -1164,6 +1281,7 @@ def ST_MinimumClearance(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_MinimumClearance", geometry)
+
@validate_argument_types
def ST_MinimumClearanceLine(geometry: ColumnOrName) -> Column:
"""Calculate the minimum clearance Linestring between two vertices
@@ -1175,8 +1293,11 @@ def ST_MinimumClearanceLine(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_MinimumClearanceLine", geometry)
+
@validate_argument_types
-def ST_MinimumBoundingCircle(geometry: ColumnOrName, quadrant_segments: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_MinimumBoundingCircle(
+ geometry: ColumnOrName, quadrant_segments: Optional[Union[ColumnOrName, int]] = None
+) -> Column:
"""Generate the minimum bounding circle that contains a geometry.
:param geometry: Geometry column to generate minimum bounding circles for.
@@ -1273,6 +1394,7 @@ def ST_NumInteriorRings(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_NumInteriorRings", geometry)
+
@validate_argument_types
def ST_NumInteriorRing(geometry: ColumnOrName) -> Column:
"""Return the number of interior rings contained in a polygon geometry.
@@ -1312,7 +1434,9 @@ def ST_PointOnSurface(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_ReducePrecision(geometry: ColumnOrName, precision: Union[ColumnOrName, int]) -> Column:
+def ST_ReducePrecision(
+ geometry: ColumnOrName, precision: Union[ColumnOrName, int]
+) -> Column:
"""Reduce the precision of the coordinates in geometry to a specified number of decimal places.
:param geometry: Geometry to reduce the precision of.
@@ -1326,7 +1450,9 @@ def ST_ReducePrecision(geometry: ColumnOrName, precision: Union[ColumnOrName, in
@validate_argument_types
-def ST_RemovePoint(line_string: ColumnOrName, index: Union[ColumnOrName, int]) -> Column:
+def ST_RemovePoint(
+ line_string: ColumnOrName, index: Union[ColumnOrName, int]
+) -> Column:
"""Remove the specified point (0-th based) for a linestring geometry column.
:param line_string: Linestring geometry column to remove the point from.
@@ -1340,7 +1466,9 @@ def ST_RemovePoint(line_string: ColumnOrName, index: Union[ColumnOrName, int]) -
@validate_argument_types
-def ST_RemoveRepeatedPoints(geom: ColumnOrName, tolerance: Optional[Union[ColumnOrName, float]] = None) -> Column:
+def ST_RemoveRepeatedPoints(
+ geom: ColumnOrName, tolerance: Optional[Union[ColumnOrName, float]] = None
+) -> Column:
"""Removes duplicate coordinates from a geometry, optionally removing those within a specified distance tolerance.
@param geom: Geometry with repeated points
@@ -1394,7 +1522,9 @@ def ST_S2ToGeom(cells: Union[ColumnOrName, list]) -> Column:
@validate_argument_types
-def ST_SetPoint(line_string: ColumnOrName, index: Union[ColumnOrName, int], point: ColumnOrName) -> Column:
+def ST_SetPoint(
+ line_string: ColumnOrName, index: Union[ColumnOrName, int], point: ColumnOrName
+) -> Column:
"""Replace a point in a linestring.
:param line_string: Linestring geometry column which contains the point to be replaced.
@@ -1422,8 +1552,11 @@ def ST_SetSRID(geometry: ColumnOrName, srid: Union[ColumnOrName, int]) -> Column
"""
return _call_st_function("ST_SetSRID", (geometry, srid))
+
@validate_argument_types
-def ST_Snap(input: ColumnOrName, reference: ColumnOrName, tolerance: Union[ColumnOrName, float]) -> Column:
+def ST_Snap(
+ input: ColumnOrName, reference: ColumnOrName, tolerance: Union[ColumnOrName, float]
+) -> Column:
"""Snaps input Geometry to reference Geometry controlled by distance tolerance.
:param input: Geometry
@@ -1481,7 +1614,9 @@ def ST_StartPoint(line_string: ColumnOrName) -> Column:
@validate_argument_types
-def ST_SubDivide(geometry: ColumnOrName, max_vertices: Union[ColumnOrName, int]) -> Column:
+def ST_SubDivide(
+ geometry: ColumnOrName, max_vertices: Union[ColumnOrName, int]
+) -> Column:
"""Subdivide a geometry into an array of geometries with at maximum number of vertices in each.
:param geometry: Geometry column to subdivide.
@@ -1495,7 +1630,9 @@ def ST_SubDivide(geometry: ColumnOrName, max_vertices: Union[ColumnOrName, int])
@validate_argument_types
-def ST_SubDivideExplode(geometry: ColumnOrName, max_vertices: Union[ColumnOrName, int]) -> Column:
+def ST_SubDivideExplode(
+ geometry: ColumnOrName, max_vertices: Union[ColumnOrName, int]
+) -> Column:
"""Same as ST_SubDivide except also explode the generated array into multiple rows.
:param geometry: Geometry column to subdivide.
@@ -1509,7 +1646,9 @@ def ST_SubDivideExplode(geometry: ColumnOrName, max_vertices: Union[ColumnOrName
@validate_argument_types
-def ST_SimplifyPreserveTopology(geometry: ColumnOrName, distance_tolerance: ColumnOrNameOrNumber) -> Column:
+def ST_SimplifyPreserveTopology(
+ geometry: ColumnOrName, distance_tolerance: ColumnOrNameOrNumber
+) -> Column:
"""Simplify a geometry within a specified tolerance while preserving topological relationships.
:param geometry: Geometry column to simplify.
@@ -1519,10 +1658,15 @@ def ST_SimplifyPreserveTopology(geometry: ColumnOrName, distance_tolerance: Colu
:return: Simplified geometry as a geometry column.
:rtype: Column
"""
- return _call_st_function("ST_SimplifyPreserveTopology", (geometry, distance_tolerance))
+ return _call_st_function(
+ "ST_SimplifyPreserveTopology", (geometry, distance_tolerance)
+ )
+
@validate_argument_types
-def ST_SimplifyVW(geometry: ColumnOrName, distance_tolerance: ColumnOrNameOrNumber) -> Column:
+def ST_SimplifyVW(
+ geometry: ColumnOrName, distance_tolerance: ColumnOrNameOrNumber
+) -> Column:
"""Simplify a geometry using Visvalingam-Whyatt algorithm within a specified tolerance while preserving topological relationships.
:param geometry: Geometry column to simplify.
@@ -1536,7 +1680,11 @@ def ST_SimplifyVW(geometry: ColumnOrName, distance_tolerance: ColumnOrNameOrNumb
@validate_argument_types
-def ST_SimplifyPolygonHull(geometry: ColumnOrName, vertexFactor: ColumnOrNameOrNumber, isOuter: Optional[Union[ColumnOrName, bool]] = None) -> Column:
+def ST_SimplifyPolygonHull(
+ geometry: ColumnOrName,
+ vertexFactor: ColumnOrNameOrNumber,
+ isOuter: Optional[Union[ColumnOrName, bool]] = None,
+) -> Column:
"""Simplify a geometry using Visvalingam-Whyatt algorithm within a specified tolerance while preserving topological relationships.
:param geometry: Geometry column to simplify.
@@ -1546,10 +1694,15 @@ def ST_SimplifyPolygonHull(geometry: ColumnOrName, vertexFactor: ColumnOrNameOrN
:return: Simplified geometry as a geometry column.
:rtype: Column
"""
- args = (geometry, vertexFactor) if isOuter is None else (geometry, vertexFactor, isOuter)
+ args = (
+ (geometry, vertexFactor)
+ if isOuter is None
+ else (geometry, vertexFactor, isOuter)
+ )
return _call_st_function("ST_SimplifyPolygonHull", args)
+
@validate_argument_types
def ST_Split(input: ColumnOrName, blade: ColumnOrName) -> Column:
"""Split input geometry by the blade geometry.
@@ -1579,7 +1732,12 @@ def ST_SymDifference(a: ColumnOrName, b: ColumnOrName) -> Column:
@validate_argument_types
-def ST_Transform(geometry: ColumnOrName, source_crs: ColumnOrName, target_crs: Optional[Union[ColumnOrName, str]] = None, disable_error: Optional[Union[ColumnOrName, bool]] = None) -> Column:
+def ST_Transform(
+ geometry: ColumnOrName,
+ source_crs: ColumnOrName,
+ target_crs: Optional[Union[ColumnOrName, str]] = None,
+ disable_error: Optional[Union[ColumnOrName, bool]] = None,
+) -> Column:
"""Convert a geometry from one coordinate system to another coordinate system.
:param geometry: Geometry column to convert.
@@ -1607,6 +1765,7 @@ def ST_Transform(geometry: ColumnOrName, source_crs: ColumnOrName, target_crs: O
args = (geometry, source_crs, target_crs, disable_error)
return _call_st_function("ST_Transform", args)
+
@validate_argument_types
def ST_TriangulatePolygon(geom: ColumnOrName) -> Column:
"""Computes the constrained Delaunay triangulation of polygons. Holes and Multipolygons are supported.
@@ -1618,17 +1777,19 @@ def ST_TriangulatePolygon(geom: ColumnOrName) -> Column:
"""
return _call_st_function("ST_TriangulatePolygon", geom)
+
@validate_argument_types
def ST_UnaryUnion(geom: ColumnOrName) -> Column:
"""Calculate the unary union of a geometry
- :param geom: Geometry to do union
- :type geom: ColumnOrName
- :return: Geometry representing the unary union of geom as a geometry column.
- :rtype: Column
- """
+ :param geom: Geometry to do union
+ :type geom: ColumnOrName
+ :return: Geometry representing the unary union of geom as a geometry column.
+ :rtype: Column
+ """
return _call_st_function("ST_UnaryUnion", geom)
+
@validate_argument_types
def ST_Union(a: ColumnOrName, b: Optional[ColumnOrName] = None) -> Column:
"""Calculate the union of two geometries.
@@ -1732,6 +1893,7 @@ def ST_Z(point: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Z", point)
+
@validate_argument_types
def ST_Zmflag(geom: ColumnOrName) -> Column:
"""Return the code indicating the ZM coordinate dimension of a geometry
@@ -1744,6 +1906,7 @@ def ST_Zmflag(geom: ColumnOrName) -> Column:
"""
return _call_st_function("ST_Zmflag", geom)
+
@validate_argument_types
def ST_ZMax(geometry: ColumnOrName) -> Column:
"""Return the maximum Z coordinate of a geometry.
@@ -1755,6 +1918,7 @@ def ST_ZMax(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ZMax", geometry)
+
@validate_argument_types
def ST_ZMin(geometry: ColumnOrName) -> Column:
"""Return the minimum Z coordinate of a geometry.
@@ -1765,6 +1929,8 @@ def ST_ZMin(geometry: ColumnOrName) -> Column:
:rtype: Column
"""
return _call_st_function("ST_ZMin", geometry)
+
+
@validate_argument_types
def ST_NumPoints(geometry: ColumnOrName) -> Column:
"""Return the number of points in a LineString
@@ -1775,8 +1941,11 @@ def ST_NumPoints(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_NumPoints", geometry)
+
@validate_argument_types
-def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column:
+def ST_Force3D(
+ geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0
+) -> Column:
"""
Return a geometry with a 3D coordinate of value 'zValue' forced upon it. No change happens if the geometry is already 3D
:param zValue: Optional value of z coordinate to be potentially added, default value is 0.0
@@ -1786,8 +1955,11 @@ def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, floa
args = (geometry, zValue)
return _call_st_function("ST_Force3D", args)
+
@validate_argument_types
-def ST_Force3DM(geometry: ColumnOrName, mValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column:
+def ST_Force3DM(
+ geometry: ColumnOrName, mValue: Optional[Union[ColumnOrName, float]] = 0.0
+) -> Column:
"""
Return a geometry with a 3D coordinate of value 'mValue' forced upon it. No change happens if the geometry is already 3D
:param mValue: Optional value of m coordinate to be potentially added, default value is 0.0
@@ -1797,8 +1969,11 @@ def ST_Force3DM(geometry: ColumnOrName, mValue: Optional[Union[ColumnOrName, flo
args = (geometry, mValue)
return _call_st_function("ST_Force3DM", args)
+
@validate_argument_types
-def ST_Force3DZ(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column:
+def ST_Force3DZ(
+ geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0
+) -> Column:
"""
Return a geometry with a 3D coordinate of value 'zValue' forced upon it. No change happens if the geometry is already 3D
:param zValue: Optional value of z coordinate to be potentially added, default value is 0.0
@@ -1808,9 +1983,13 @@ def ST_Force3DZ(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, flo
args = (geometry, zValue)
return _call_st_function("ST_Force3DZ", args)
+
@validate_argument_types
-def ST_Force4D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0,
- mValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column:
+def ST_Force4D(
+ geometry: ColumnOrName,
+ zValue: Optional[Union[ColumnOrName, float]] = 0.0,
+ mValue: Optional[Union[ColumnOrName, float]] = 0.0,
+) -> Column:
"""
Return a geometry with a 4D coordinate of value 'zValue' and mValue forced upon it. No change happens if the
geometry is already 4D, if geometry either has z or m, it will not change the existing z or m value.
@@ -1822,6 +2001,7 @@ def ST_Force4D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, floa
args = (geometry, zValue, mValue)
return _call_st_function("ST_Force4D", args)
+
@validate_argument_types
def ST_ForceCollection(geometry: ColumnOrName) -> Column:
"""
@@ -1832,6 +2012,7 @@ def ST_ForceCollection(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ForceCollection", geometry)
+
@validate_argument_types
def ST_ForcePolygonCW(geometry: ColumnOrName) -> Column:
"""
@@ -1841,6 +2022,7 @@ def ST_ForcePolygonCW(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ForcePolygonCW", geometry)
+
@validate_argument_types
def ST_ForceRHR(geometry: ColumnOrName) -> Column:
"""
@@ -1850,6 +2032,7 @@ def ST_ForceRHR(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_ForceRHR", geometry)
+
@validate_argument_types
def ST_NRings(geometry: ColumnOrName) -> Column:
"""
@@ -1858,8 +2041,15 @@ def ST_NRings(geometry: ColumnOrName) -> Column:
:return: Number of exterior rings + interior rings (if any) for the given Polygon or MultiPolygon
"""
return _call_st_function("ST_NRings", geometry)
+
+
@validate_argument_types
-def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], deltaY: Union[ColumnOrName, float], deltaZ: Optional[Union[ColumnOrName, float]] = 0.0) -> Column:
+def ST_Translate(
+ geometry: ColumnOrName,
+ deltaX: Union[ColumnOrName, float],
+ deltaY: Union[ColumnOrName, float],
+ deltaZ: Optional[Union[ColumnOrName, float]] = 0.0,
+) -> Column:
"""
Returns the geometry with x, y and z (if present) coordinates offset by given deltaX, deltaY, and deltaZ values.
:param geometry: Geometry column whose coordinates are to be translated.
@@ -1871,8 +2061,13 @@ def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], del
args = (geometry, deltaX, deltaY, deltaZ)
return _call_st_function("ST_Translate", args)
+
@validate_argument_types
-def ST_VoronoiPolygons(geometry: ColumnOrName, tolerance: Optional[Union[ColumnOrName, float]] = 0.0, extendTo: Optional[ColumnOrName] = None) -> Column:
+def ST_VoronoiPolygons(
+ geometry: ColumnOrName,
+ tolerance: Optional[Union[ColumnOrName, float]] = 0.0,
+ extendTo: Optional[ColumnOrName] = None,
+) -> Column:
"""
ST_VoronoiPolygons computes a two-dimensional Voronoi diagram from the vertices of the supplied geometry.
The result is a GeometryCollection of Polygons that covers an envelope larger than the extent of the input vertices.
@@ -1887,6 +2082,7 @@ def ST_VoronoiPolygons(geometry: ColumnOrName, tolerance: Optional[Union[ColumnO
args = (geometry, tolerance, extendTo)
return _call_st_function("ST_VoronoiPolygons", args)
+
@validate_argument_types
def ST_FrechetDistance(g1: ColumnOrName, g2: ColumnOrName) -> Column:
"""
@@ -1900,11 +2096,23 @@ def ST_FrechetDistance(g1: ColumnOrName, g2: ColumnOrName) -> Column:
args = (g1, g2)
return _call_st_function("ST_FrechetDistance", args)
+
@validate_argument_types
-def ST_Affine(geometry: ColumnOrName, a: Union[ColumnOrName, float], b: Union[ColumnOrName, float], d: Union[ColumnOrName, float],
- e: Union[ColumnOrName, float], xOff: Union[ColumnOrName, float], yOff: Union[ColumnOrName, float], c: Optional[Union[ColumnOrName, float]] = None, f: Optional[Union[ColumnOrName, float]] = None,
- g: Optional[Union[ColumnOrName, float]] = None, h: Optional[Union[ColumnOrName, float]] = None,
- i: Optional[Union[ColumnOrName, float]] = None, zOff: Optional[Union[ColumnOrName, float]] = None) -> Column:
+def ST_Affine(
+ geometry: ColumnOrName,
+ a: Union[ColumnOrName, float],
+ b: Union[ColumnOrName, float],
+ d: Union[ColumnOrName, float],
+ e: Union[ColumnOrName, float],
+ xOff: Union[ColumnOrName, float],
+ yOff: Union[ColumnOrName, float],
+ c: Optional[Union[ColumnOrName, float]] = None,
+ f: Optional[Union[ColumnOrName, float]] = None,
+ g: Optional[Union[ColumnOrName, float]] = None,
+ h: Optional[Union[ColumnOrName, float]] = None,
+ i: Optional[Union[ColumnOrName, float]] = None,
+ zOff: Optional[Union[ColumnOrName, float]] = None,
+) -> Column:
"""
Apply a 3D/2D affine transformation to the given geometry
x = a * x + b * y + c * z + xOff | x = a * x + b * y + xOff
@@ -1930,6 +2138,7 @@ def ST_Affine(geometry: ColumnOrName, a: Union[ColumnOrName, float], b: Union[Co
args = (geometry, a, b, c, d, e, f, g, h, i, xOff, yOff, zOff)
return _call_st_function("ST_Affine", args)
+
@validate_argument_types
def ST_BoundingDiagonal(geometry: ColumnOrName) -> Column:
"""
@@ -1943,7 +2152,12 @@ def ST_BoundingDiagonal(geometry: ColumnOrName) -> Column:
@validate_argument_types
-def ST_Angle(g1: ColumnOrName, g2: ColumnOrName, g3: Optional[ColumnOrName] = None, g4: Optional[ColumnOrName] = None) -> Column:
+def ST_Angle(
+ g1: ColumnOrName,
+ g2: ColumnOrName,
+ g3: Optional[ColumnOrName] = None,
+ g4: Optional[ColumnOrName] = None,
+) -> Column:
"""
Returns the computed angle between vectors formed by given geometries in radian. Range of result is between 0 and 2 * pi.
3 Variants:
@@ -1968,6 +2182,7 @@ def ST_Angle(g1: ColumnOrName, g2: ColumnOrName, g3: Optional[ColumnOrName] = No
# args = (g1, g2, g3, g4)
return _call_st_function("ST_Angle", args)
+
@validate_argument_types
def ST_Degrees(angleInRadian: Union[ColumnOrName, float]) -> Column:
"""
@@ -1977,8 +2192,13 @@ def ST_Degrees(angleInRadian: Union[ColumnOrName, float]) -> Column:
"""
return _call_st_function("ST_Degrees", angleInRadian)
+
@validate_argument_types
-def ST_DelaunayTriangles(geometry: ColumnOrName, tolerance: Optional[Union[ColumnOrName, float]] = None, flag: Optional[Union[ColumnOrName, int]] = None) -> Column:
+def ST_DelaunayTriangles(
+ geometry: ColumnOrName,
+ tolerance: Optional[Union[ColumnOrName, float]] = None,
+ flag: Optional[Union[ColumnOrName, int]] = None,
+) -> Column:
"""
Computes the Delaunay Triangles of the vertices of the input geometry.
@@ -1993,15 +2213,20 @@ def ST_DelaunayTriangles(geometry: ColumnOrName, tolerance: Optional[Union[Colum
"""
if flag is None and tolerance is None:
- args = (geometry)
+ args = geometry
elif flag is None:
args = (geometry, tolerance)
else:
args = (geometry, tolerance, flag)
return _call_st_function("ST_DelaunayTriangles", args)
+
@validate_argument_types
-def ST_HausdorffDistance(g1: ColumnOrName, g2: ColumnOrName, densityFrac: Optional[Union[ColumnOrName, float]] = -1) -> Column:
+def ST_HausdorffDistance(
+ g1: ColumnOrName,
+ g2: ColumnOrName,
+ densityFrac: Optional[Union[ColumnOrName, float]] = -1,
+) -> Column:
"""
Returns discretized (and hence approximate) hausdorff distance between two given geometries.
Optionally, a distance fraction can also be provided which decreases the gap between actual and discretized hausforff distance
@@ -2013,6 +2238,7 @@ def ST_HausdorffDistance(g1: ColumnOrName, g2: ColumnOrName, densityFrac: Option
args = (g1, g2, densityFrac)
return _call_st_function("ST_HausdorffDistance", args)
+
@validate_argument_types
def ST_CoordDim(geometry: ColumnOrName) -> Column:
"""Return the number of dimensions contained in a coordinate
@@ -2024,6 +2250,7 @@ def ST_CoordDim(geometry: ColumnOrName) -> Column:
"""
return _call_st_function("ST_CoordDim", geometry)
+
@validate_argument_types
def ST_IsCollection(geometry: ColumnOrName) -> Column:
"""Check if the geometry is of GeometryCollection type.
@@ -2065,8 +2292,13 @@ def ST_RotateY(geometry: ColumnOrName, angle: Union[ColumnOrName, float]) -> Col
@validate_argument_types
-def ST_Rotate(geometry: ColumnOrName, angle: Union[ColumnOrName, float], originX: Union[ColumnOrName, float] = None,
- originY: Union[ColumnOrName, float] = None, pointOrigin: ColumnOrName = None) -> Column:
+def ST_Rotate(
+ geometry: ColumnOrName,
+ angle: Union[ColumnOrName, float],
+ originX: Union[ColumnOrName, float] = None,
+ originY: Union[ColumnOrName, float] = None,
+ pointOrigin: ColumnOrName = None,
+) -> Column:
"""Return a counter-clockwise rotated geometry along the specified origin.
:param geometry: Geometry column or name.
@@ -2093,5 +2325,8 @@ def ST_Rotate(geometry: ColumnOrName, angle: Union[ColumnOrName, float], originX
# Automatically populate __all__
-__all__ = [name for name, obj in inspect.getmembers(sys.modules[__name__])
- if inspect.isfunction(obj) and name != 'GeometryType']
+__all__ = [
+ name
+ for name, obj in inspect.getmembers(sys.modules[__name__])
+ if inspect.isfunction(obj) and name != "GeometryType"
+]
diff --git a/python/sedona/sql/st_predicates.py b/python/sedona/sql/st_predicates.py
index 32efd7c5c..b91edf8ee 100644
--- a/python/sedona/sql/st_predicates.py
+++ b/python/sedona/sql/st_predicates.py
@@ -22,7 +22,11 @@ from functools import partial
from pyspark.sql import Column
from typing import Union, Optional
-from sedona.sql.dataframe_api import ColumnOrName, call_sedona_function, validate_argument_types
+from sedona.sql.dataframe_api import (
+ ColumnOrName,
+ call_sedona_function,
+ validate_argument_types,
+)
_call_predicate_function = partial(call_sedona_function, "st_predicates")
@@ -55,6 +59,7 @@ def ST_Crosses(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_predicate_function("ST_Crosses", (a, b))
+
@validate_argument_types
def ST_Disjoint(a: ColumnOrName, b: ColumnOrName) -> Column:
"""Check whether two geometries are disjoint.
@@ -138,8 +143,11 @@ def ST_Touches(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_predicate_function("ST_Touches", (a, b))
+
@validate_argument_types
-def ST_Relate(a: ColumnOrName, b: ColumnOrName, intersectionMatrix: Optional[ColumnOrName] = None) -> Column:
+def ST_Relate(
+ a: ColumnOrName, b: ColumnOrName, intersectionMatrix: Optional[ColumnOrName] = None
+) -> Column:
"""Check whether two geometries are related to each other.
:param a: One geometry column to check.
@@ -155,6 +163,7 @@ def ST_Relate(a: ColumnOrName, b: ColumnOrName, intersectionMatrix: Optional[Col
return _call_predicate_function("ST_Relate", args)
+
@validate_argument_types
def ST_RelateMatch(matrix1: ColumnOrName, matrix2: ColumnOrName) -> Column:
"""Check whether two DE-9IM are related to each other.
@@ -210,8 +219,14 @@ def ST_CoveredBy(a: ColumnOrName, b: ColumnOrName) -> Column:
"""
return _call_predicate_function("ST_CoveredBy", (a, b))
+
@validate_argument_types
-def ST_DWithin(a: ColumnOrName, b: ColumnOrName, distance: Union[ColumnOrName, float], use_sphere: Optional[Union[ColumnOrName, bool]] = None):
+def ST_DWithin(
+ a: ColumnOrName,
+ b: ColumnOrName,
+ distance: Union[ColumnOrName, float],
+ use_sphere: Optional[Union[ColumnOrName, bool]] = None,
+):
"""
Check if geometry a is within 'distance' units of geometry b
:param a: Geometry column to check
@@ -220,10 +235,21 @@ def ST_DWithin(a: ColumnOrName, b: ColumnOrName, distance: Union[ColumnOrName, f
:param use_sphere: whether to use spheroid distance or euclidean distance
:return: True if a is within distance units of Geometry b
"""
- args = (a, b, distance, use_sphere) if use_sphere is not None else (a, b, distance,)
+ args = (
+ (a, b, distance, use_sphere)
+ if use_sphere is not None
+ else (
+ a,
+ b,
+ distance,
+ )
+ )
return _call_predicate_function("ST_DWithin", args)
# Automatically populate __all__
-__all__ = [name for name, obj in inspect.getmembers(sys.modules[__name__])
- if inspect.isfunction(obj)]
+__all__ = [
+ name
+ for name, obj in inspect.getmembers(sys.modules[__name__])
+ if inspect.isfunction(obj)
+]
diff --git a/python/sedona/utils/abstract_parser.py b/python/sedona/utils/abstract_parser.py
index 2679a26f6..fdf3a067c 100644
--- a/python/sedona/utils/abstract_parser.py
+++ b/python/sedona/utils/abstract_parser.py
@@ -29,9 +29,9 @@ class GeometryParser(ABC):
raise NotImplementedError
@classmethod
- def serialize(cls, obj: BaseGeometry, binary_buffer: 'BinaryBuffer'):
+ def serialize(cls, obj: BaseGeometry, binary_buffer: "BinaryBuffer"):
raise NotImplementedError("Parser has to implement serialize method")
@classmethod
- def deserialize(cls, bin_parser: 'BinaryParser') -> BaseGeometry:
+ def deserialize(cls, bin_parser: "BinaryParser") -> BaseGeometry:
raise NotImplementedError("Parser has to implement deserialize method")
diff --git a/python/sedona/utils/adapter.py b/python/sedona/utils/adapter.py
index 5fab3de79..a7d786e5e 100644
--- a/python/sedona/utils/adapter.py
+++ b/python/sedona/utils/adapter.py
@@ -33,17 +33,17 @@ class Adapter(metaclass=MultipleMeta):
@staticmethod
def _create_dataframe(jdf, sparkSession: SparkSession) -> DataFrame:
- if hasattr(sparkSession, '_wrapped'):
+ if hasattr(sparkSession, "_wrapped"):
# In Spark < 3.3, use the _wrapped SQLContext
return DataFrame(jdf, sparkSession._wrapped)
else:
# In Spark >= 3.3, use the session directly
return DataFrame(jdf, sparkSession)
-
@classmethod
- def toRdd(cls, dataFrame: DataFrame) -> 'JvmSpatialRDD':
+ def toRdd(cls, dataFrame: DataFrame) -> "JvmSpatialRDD":
from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD
+
sc = dataFrame._sc
jvm = sc._jvm
@@ -70,7 +70,9 @@ class Adapter(metaclass=MultipleMeta):
return spatial_rdd
@classmethod
- def toSpatialRdd(cls, dataFrame: DataFrame, geometryFieldName: str, fieldNames: List) -> SpatialRDD:
+ def toSpatialRdd(
+ cls, dataFrame: DataFrame, geometryFieldName: str, fieldNames: List
+ ) -> SpatialRDD:
"""
:param dataFrame:
@@ -81,7 +83,9 @@ class Adapter(metaclass=MultipleMeta):
sc = dataFrame._sc
jvm = sc._jvm
- srdd = jvm.PythonAdapterWrapper.toSpatialRdd(dataFrame._jdf, geometryFieldName, fieldNames)
+ srdd = jvm.PythonAdapterWrapper.toSpatialRdd(
+ dataFrame._jdf, geometryFieldName, fieldNames
+ )
spatial_rdd = SpatialRDD(sc)
spatial_rdd.set_srdd(srdd)
@@ -89,7 +93,9 @@ class Adapter(metaclass=MultipleMeta):
return spatial_rdd
@classmethod
- def toDf(cls, spatialRDD: SpatialRDD, fieldNames: List, sparkSession: SparkSession) -> DataFrame:
+ def toDf(
+ cls, spatialRDD: SpatialRDD, fieldNames: List, sparkSession: SparkSession
+ ) -> DataFrame:
"""
:param spatialRDD:
@@ -100,7 +106,9 @@ class Adapter(metaclass=MultipleMeta):
sc = spatialRDD._sc
jvm = sc._jvm
- jdf = jvm.PythonAdapterWrapper.toDf(spatialRDD._srdd, fieldNames, sparkSession._jsparkSession)
+ jdf = jvm.PythonAdapterWrapper.toDf(
+ spatialRDD._srdd, fieldNames, sparkSession._jsparkSession
+ )
df = Adapter._create_dataframe(jdf, sparkSession)
@@ -132,13 +140,24 @@ class Adapter(metaclass=MultipleMeta):
:return:
"""
spatial_pair_rdd_mapped = spatialPairRDD.map(
- lambda x: [x[0].geom, *x[0].getUserData().split("\t"), x[1].geom, *x[1].getUserData().split("\t")]
+ lambda x: [
+ x[0].geom,
+ *x[0].getUserData().split("\t"),
+ x[1].geom,
+ *x[1].getUserData().split("\t"),
+ ]
)
df = sparkSession.createDataFrame(spatial_pair_rdd_mapped)
return df
@classmethod
- def toDf(cls, spatialPairRDD: RDD, leftFieldnames: List, rightFieldNames: List, sparkSession: SparkSession):
+ def toDf(
+ cls,
+ spatialPairRDD: RDD,
+ leftFieldnames: List,
+ rightFieldNames: List,
+ sparkSession: SparkSession,
+ ):
"""
:param spatialPairRDD:
@@ -164,15 +183,27 @@ class Adapter(metaclass=MultipleMeta):
return df
@classmethod
- def toDf(cls, rawPairRDD: SedonaPairRDD, leftFieldnames: List, rightFieldNames: List, sparkSession: SparkSession):
+ def toDf(
+ cls,
+ rawPairRDD: SedonaPairRDD,
+ leftFieldnames: List,
+ rightFieldNames: List,
+ sparkSession: SparkSession,
+ ):
jvm = sparkSession._jvm
jdf = jvm.PythonAdapterWrapper.toDf(
- rawPairRDD.jsrdd, leftFieldnames, rightFieldNames, sparkSession._jsparkSession)
+ rawPairRDD.jsrdd,
+ leftFieldnames,
+ rightFieldNames,
+ sparkSession._jsparkSession,
+ )
df = Adapter._create_dataframe(jdf, sparkSession)
return df
@classmethod
- def toDf(cls, spatialRDD: SedonaRDD, spark: SparkSession, fieldNames: List = None) -> DataFrame:
+ def toDf(
+ cls, spatialRDD: SedonaRDD, spark: SparkSession, fieldNames: List = None
+ ) -> DataFrame:
srdd = SpatialRDD(spatialRDD.sc)
srdd.setRawSpatialRDD(spatialRDD.jsrdd)
if fieldNames:
diff --git a/python/sedona/utils/binary_parser.py b/python/sedona/utils/binary_parser.py
index d680421b1..473a63430 100644
--- a/python/sedona/utils/binary_parser.py
+++ b/python/sedona/utils/binary_parser.py
@@ -33,7 +33,7 @@ size_dict = {
"i": INT_SIZE,
"b": BYTE_SIZE,
"s": CHAR_SIZE,
- "?": BOOLEAN_SIZE
+ "?": BOOLEAN_SIZE,
}
@@ -47,8 +47,12 @@ class BinaryParser:
self.bytes = self._convert_to_binary_array(no_negatives)
def read_geometry(self, length: int):
- geom_bytes = b"".join([struct.pack("b", el) if el < 128 else struct.pack("b", el - 256) for el in
- self.bytes[self.current_index: self.current_index + length]])
+ geom_bytes = b"".join(
+ [
+ struct.pack("b", el) if el < 128 else struct.pack("b", el - 256)
+ for el in self.bytes[self.current_index : self.current_index + length]
+ ]
+ )
geom = loads(geom_bytes)
self.current_index += length
return geom
@@ -84,7 +88,7 @@ class BinaryParser:
return data
def read_string(self, length: int, encoding: str = "utf8"):
- string = self.bytes[self.current_index: self.current_index + length]
+ string = self.bytes[self.current_index : self.current_index + length]
self.current_index += length
try:
@@ -97,7 +101,7 @@ class BinaryParser:
array_length = length - self.current_index
byte_array = sc._gateway.new_array(sc._jvm.Byte, array_length)
- for index, bt in enumerate(self.bytes[self.current_index: length]):
+ for index, bt in enumerate(self.bytes[self.current_index : length]):
byte_array[index] = self.bytes[self.current_index + index]
decoded_string = sc._jvm.org.imbruced.geo_pyspark.serializers.GeoSerializerData.deserializeUserData(
byte_array
@@ -107,12 +111,16 @@ class BinaryParser:
def unpack(self, tp: str, bytes: bytearray):
max_index = self.current_index + size_dict[tp]
- bytes = self._convert_to_binary_array(bytes[self.current_index: max_index])
+ bytes = self._convert_to_binary_array(bytes[self.current_index : max_index])
return struct.unpack(tp, bytes)[0]
def unpack_reverse(self, tp: str, bytes: bytearray):
max_index = self.current_index + size_dict[tp]
- bytes = bytearray(reversed(self._convert_to_binary_array(bytes[self.current_index: max_index])))
+ bytes = bytearray(
+ reversed(
+ self._convert_to_binary_array(bytes[self.current_index : max_index])
+ )
+ )
return struct.unpack(tp, bytes)[0]
@classmethod
diff --git a/python/sedona/utils/decorators.py b/python/sedona/utils/decorators.py
index 1b59db3ae..9d10212b3 100644
--- a/python/sedona/utils/decorators.py
+++ b/python/sedona/utils/decorators.py
@@ -17,7 +17,7 @@
from typing import List, Iterable, Callable, TypeVar
-T = TypeVar('T')
+T = TypeVar("T")
class classproperty(object):
@@ -32,7 +32,9 @@ class classproperty(object):
return self.f()
-def get_first_meet_criteria_element_from_iterable(iterable: Iterable[T], criteria: Callable[[T], int]) -> int:
+def get_first_meet_criteria_element_from_iterable(
+ iterable: Iterable[T], criteria: Callable[[T], int]
+) -> int:
for index, element in enumerate(iterable):
if criteria(element):
return index
@@ -43,6 +45,7 @@ def require(library_names: List[str]):
def wrapper(func):
def run_function(*args, **kwargs):
from sedona.core.utils import ImportedJvmLib
+
has_all_libs = [lib for lib in library_names]
first_not_fulfill_value = get_first_meet_criteria_element_from_iterable(
has_all_libs, lambda x: not ImportedJvmLib.has_library(x)
@@ -53,7 +56,8 @@ def require(library_names: List[str]):
else:
raise ModuleNotFoundError(
f"Did not found {has_all_libs[first_not_fulfill_value]}, make sure that was correctly imported via py4j"
- f"Did you use SedonaRegistrator.registerAll, Your jars were properly copied to $SPARK_HOME/jars ? ")
+ f"Did you use SedonaRegistrator.registerAll, Your jars were properly copied to $SPARK_HOME/jars ? "
+ )
return run_function
diff --git a/python/sedona/utils/geometry_serde.py b/python/sedona/utils/geometry_serde.py
index 32a646cbe..cf872dfb2 100644
--- a/python/sedona/utils/geometry_serde.py
+++ b/python/sedona/utils/geometry_serde.py
@@ -34,19 +34,24 @@ try:
def find_geos_c_dll():
packages_dir = os.path.dirname(os.path.dirname(shapely.__file__))
- for lib_dirname in ['shapely.libs', 'Shapely.libs']:
+ for lib_dirname in ["shapely.libs", "Shapely.libs"]:
lib_dirpath = os.path.join(packages_dir, lib_dirname)
if not os.path.exists(lib_dirpath):
continue
for filename in os.listdir(lib_dirpath):
- if filename.lower().startswith('geos_c') and filename.lower().endswith('.dll'):
+ if filename.lower().startswith("geos_c") and filename.lower().endswith(
+ ".dll"
+ ):
return os.path.join(lib_dirpath, filename)
- raise RuntimeError('geos_c DLL not found in {}\\[S|s]hapely.libs'.format(packages_dir))
+ raise RuntimeError(
+ "geos_c DLL not found in {}\\[S|s]hapely.libs".format(packages_dir)
+ )
- if shapely.__version__.startswith('2.'):
- if sys.platform != 'win32':
+ if shapely.__version__.startswith("2."):
+ if sys.platform != "win32":
# We load geos_c library indirectly by loading shapely.lib
import shapely.lib
+
geomserde_speedup.load_libgeos_c(shapely.lib.__file__)
else:
# Find geos_c library and load it
@@ -62,7 +67,7 @@ try:
speedup_enabled = True
- elif shapely.__version__.startswith('1.'):
+ elif shapely.__version__.startswith("1."):
# Shapely 1.x uses ctypes.CDLL to load geos_c library. We can obtain the
# handle of geos_c library from `shapely.geos._lgeos._handle`
import shapely.geos
@@ -75,7 +80,7 @@ try:
MultiPoint,
MultiLineString,
MultiPolygon,
- GeometryCollection
+ GeometryCollection,
)
lgeos_handle = shapely.geos._lgeos._handle
@@ -112,13 +117,13 @@ try:
ob = BaseGeometry()
geom_type = shapely.geometry.base.GEOMETRY_TYPES[geom_type_id]
ob.__class__ = GEOMETRY_CLASSES[geom_type_id]
- ob.__dict__['__geom__'] = g
- ob.__dict__['__p__'] = None
+ ob.__dict__["__geom__"] = g
+ ob.__dict__["__p__"] = None
if has_z != 0:
- ob.__dict__['_ndim'] = 3
+ ob.__dict__["_ndim"] = 3
else:
- ob.__dict__['_ndim'] = 2
- ob.__dict__['_is_empty'] = False
+ ob.__dict__["_ndim"] = 2
+ ob.__dict__["_is_empty"] = False
return ob, bytes_read
speedup_enabled = True
@@ -128,5 +133,7 @@ try:
from .geometry_serde_general import serialize, deserialize
except Exception as e:
- warn(f'Cannot load geomserde_speedup, fallback to general python implementation. Reason: {e}')
+ warn(
+ f"Cannot load geomserde_speedup, fallback to general python implementation. Reason: {e}"
+ )
from .geometry_serde_general import serialize, deserialize
diff --git a/python/sedona/utils/geometry_serde_general.py b/python/sedona/utils/geometry_serde_general.py
index 243a1f1f6..212a302b6 100644
--- a/python/sedona/utils/geometry_serde_general.py
+++ b/python/sedona/utils/geometry_serde_general.py
@@ -36,8 +36,14 @@ from shapely.wkb import dumps as wkb_dumps
from shapely.wkt import loads as wkt_loads
-CoordType = Union[Tuple[float, float], Tuple[float, float, float], Tuple[float, float, float, float]]
-ListCoordType = Union[List[Tuple[float, float]], List[Tuple[float, float, float]], List[Tuple[float, float, float, float]]]
+CoordType = Union[
+ Tuple[float, float], Tuple[float, float, float], Tuple[float, float, float, float]
+]
+ListCoordType = Union[
+ List[Tuple[float, float]],
+ List[Tuple[float, float, float]],
+ List[Tuple[float, float, float, float]],
+]
GET_COORDS_NUMPY_THRESHOLD = 50
@@ -46,6 +52,7 @@ class GeometryTypeID:
"""
Constants used to identify the geometry type in the serialized bytearray of geometry.
"""
+
POINT = 1
LINESTRING = 2
POLYGON = 3
@@ -59,6 +66,7 @@ class CoordinateType:
"""
Constants used to identify geometry dimensions in the serialized bytearray of geometry.
"""
+
XY = 1
XYZ = 2
XYM = 3
@@ -66,7 +74,7 @@ class CoordinateType:
BYTES_PER_COORDINATE = [16, 24, 24, 32]
NUM_COORD_COMPONENTS = [2, 3, 3, 4]
- UNPACK_FORMAT = ['dd', 'ddd', 'ddxxxxxxxx', 'dddxxxxxxxx']
+ UNPACK_FORMAT = ["dd", "ddd", "ddxxxxxxxx", "dddxxxxxxxx"]
@staticmethod
def type_of(geom) -> int:
@@ -99,12 +107,8 @@ class GeometryBuffer:
ints_offset: int
def __init__(
- self,
- buffer: bytearray,
- coord_type: int,
- coords_offset: int,
- num_coords: int
- ) -> None:
+ self, buffer: bytearray, coord_type: int, coords_offset: int, num_coords: int
+ ) -> None:
self.buffer = buffer
self.coord_type = coord_type
self.bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
@@ -126,17 +130,16 @@ class GeometryBuffer:
if num_rings == 0:
return Polygon()
- rings = [
- self.read_coordinates(self.read_int())
- for _ in range(num_rings)
- ]
+ rings = [self.read_coordinates(self.read_int()) for _ in range(num_rings)]
return Polygon(rings[0], rings[1:])
def write_linestring(self, line: LineString) -> None:
coords = [tuple(c) for c in line.coords]
self.write_int(len(coords))
- self.coords_offset = put_coordinates(self.buffer, self.coords_offset, self.coord_type, coords)
+ self.coords_offset = put_coordinates(
+ self.buffer, self.coords_offset, self.coord_type, coords
+ )
def write_polygon(self, polygon: Polygon) -> None:
exterior = polygon.exterior
@@ -149,7 +152,9 @@ class GeometryBuffer:
self.write_linestring(interior)
def read_coordinates(self, num_coords: int) -> ListCoordType:
- coords = get_coordinates(self.buffer, self.coords_offset, self.coord_type, num_coords)
+ coords = get_coordinates(
+ self.buffer, self.coords_offset, self.coord_type, num_coords
+ )
self.coords_offset += num_coords * self.bytes_per_coord
return coords
@@ -161,7 +166,7 @@ class GeometryBuffer:
def read_int(self) -> int:
value = struct.unpack_from("i", self.buffer, self.ints_offset)[0]
if value > len(self.buffer):
- raise ValueError('Unexpected large integer in structural data')
+ raise ValueError("Unexpected large integer in structural data")
self.ints_offset += 4
return value
@@ -169,6 +174,7 @@ class GeometryBuffer:
struct.pack_into("i", self.buffer, self.ints_offset, value)
self.ints_offset += 4
+
def serialize(geom: BaseGeometry) -> Optional[Union[bytes, bytearray]]:
"""
Serialize a shapely geometry object to the internal representation of GeometryUDT.
@@ -197,6 +203,7 @@ def serialize(geom: BaseGeometry) -> Optional[Union[bytes, bytearray]]:
else:
raise ValueError(f"Unsupported geometry type: {type(geom)}")
+
def deserialize(buffer: bytes) -> Optional[BaseGeometry]:
"""
Deserialize a shapely geometry object from the internal representation of GeometryUDT.
@@ -208,9 +215,9 @@ def deserialize(buffer: bytes) -> Optional[BaseGeometry]:
preamble_byte = buffer[0]
geom_type = (preamble_byte >> 4) & 0x0F
coord_type = (preamble_byte >> 1) & 0x07
- num_coords = struct.unpack_from('i', buffer, 4)[0]
+ num_coords = struct.unpack_from("i", buffer, 4)[0]
if num_coords > len(buffer):
- raise ValueError('num_coords cannot be larger than buffer size')
+ raise ValueError("num_coords cannot be larger than buffer size")
geom_buffer = GeometryBuffer(buffer, coord_type, 8, num_coords)
if geom_type == GeometryTypeID.POINT:
geom = deserialize_point(geom_buffer)
@@ -231,47 +238,57 @@ def deserialize(buffer: bytes) -> Optional[BaseGeometry]:
return geom, geom_buffer.ints_offset
-def create_buffer_for_geom(geom_type: int, coord_type: int, size: int, num_coords: int) -> bytearray:
+def create_buffer_for_geom(
+ geom_type: int, coord_type: int, size: int, num_coords: int
+) -> bytearray:
buffer = bytearray(size)
preamble_byte = (geom_type << 4) | (coord_type << 1)
buffer[0] = preamble_byte
- struct.pack_into('i', buffer, 4, num_coords)
+ struct.pack_into("i", buffer, 4, num_coords)
return buffer
+
def generate_header_bytes(geom_type: int, coord_type: int, num_coords: int) -> bytes:
preamble_byte = (geom_type << 4) | (coord_type << 1)
- return struct.pack(
- 'BBBBi',
- preamble_byte,
- 0,
- 0,
- 0,
- num_coords
- )
+ return struct.pack("BBBBi", preamble_byte, 0, 0, 0, num_coords)
-def put_coordinates(buffer: bytearray, offset: int, coord_type: int, coords: ListCoordType) -> int:
+def put_coordinates(
+ buffer: bytearray, offset: int, coord_type: int, coords: ListCoordType
+) -> int:
for coord in coords:
- struct.pack_into(CoordinateType.unpack_format(coord_type, buffer, offset, *coord))
+ struct.pack_into(
+ CoordinateType.unpack_format(coord_type, buffer, offset, *coord)
+ )
offset += CoordinateType.bytes_per_coord(coord_type)
return offset
-def put_coordinate(buffer: bytearray, offset: int, coord_type: int, coord: CoordType) -> int:
+def put_coordinate(
+ buffer: bytearray, offset: int, coord_type: int, coord: CoordType
+) -> int:
struct.pack_into(CoordinateType.unpack_format(coord_type, buffer, offset, *coord))
offset += CoordinateType.bytes_per_coord(coord_type)
return offset
-def get_coordinates(buffer: bytearray, offset: int, coord_type: int, num_coords: int) -> Union[np.ndarray, ListCoordType]:
+def get_coordinates(
+ buffer: bytearray, offset: int, coord_type: int, num_coords: int
+) -> Union[np.ndarray, ListCoordType]:
if num_coords < GET_COORDS_NUMPY_THRESHOLD:
coords = [
- struct.unpack_from(CoordinateType.unpack_format(coord_type), buffer, offset + (i * CoordinateType.bytes_per_coord(coord_type)))
+ struct.unpack_from(
+ CoordinateType.unpack_format(coord_type),
+ buffer,
+ offset + (i * CoordinateType.bytes_per_coord(coord_type)),
+ )
for i in range(num_coords)
]
else:
nums_per_coord = CoordinateType.components_per_coord(coord_type)
- coords = np.frombuffer(buffer, np.float64, num_coords * nums_per_coord, offset).reshape((num_coords, nums_per_coord))
+ coords = np.frombuffer(
+ buffer, np.float64, num_coords * nums_per_coord, offset
+ ).reshape((num_coords, nums_per_coord))
return coords
@@ -290,26 +307,12 @@ def serialize_point(geom: Point) -> bytes:
# FIXME this does not handle M yet, but geom.has_z is extremely slow
pack_format = "BBBBi" + "d" * geom._ndim
coord_type = CoordinateType.type_of(geom)
- preamble_byte = ((GeometryTypeID.POINT << 4) | (coord_type << 1))
+ preamble_byte = (GeometryTypeID.POINT << 4) | (coord_type << 1)
coords = coords[0]
- return struct.pack(
- pack_format,
- preamble_byte,
- 0,
- 0,
- 0,
- 1,
- *coords
- )
+ return struct.pack(pack_format, preamble_byte, 0, 0, 0, 1, *coords)
else:
- return struct.pack(
- 'BBBBi',
- 18,
- 0,
- 0,
- 0,
- 0
- )
+ return struct.pack("BBBBi", 18, 0, 0, 0, 0)
+
def deserialize_point(geom_buffer: GeometryBuffer) -> Point:
if geom_buffer.num_coords == 0:
@@ -338,7 +341,7 @@ def serialize_multi_point(geom: MultiPoint) -> bytes:
for k in range(geom._ndim):
coords.append(math.nan)
- body = array.array('d', coords).tobytes()
+ body = array.array("d", coords).tobytes()
return header + body
@@ -361,8 +364,10 @@ def serialize_linestring(geom: LineString) -> bytes:
coords = [tuple(c) for c in geom.coords]
if coords:
coord_type = CoordinateType.type_of(geom)
- header = generate_header_bytes(GeometryTypeID.LINESTRING, coord_type, len(coords))
- return header + array.array('d', [x for c in coords for x in c]).tobytes()
+ header = generate_header_bytes(
+ GeometryTypeID.LINESTRING, coord_type, len(coords)
+ )
+ return header + array.array("d", [x for c in coords for x in c]).tobytes()
else:
return generate_header_bytes(GeometryTypeID.LINESTRING, 1, 0)
@@ -382,10 +387,14 @@ def serialize_multi_linestring(geom: MultiLineString) -> bytes:
line_lengths = [len(line) for line in lines]
num_coords = sum(line_lengths)
- header = generate_header_bytes(GeometryTypeID.MULTILINESTRING, coord_type, num_coords)
- coord_data = array.array('d', [c for line in lines for coord in line for c in coord]).tobytes()
- num_lines = struct.pack('i', len(lines))
- structure_data = array.array('i', line_lengths).tobytes()
+ header = generate_header_bytes(
+ GeometryTypeID.MULTILINESTRING, coord_type, num_coords
+ )
+ coord_data = array.array(
+ "d", [c for line in lines for coord in line for c in coord]
+ ).tobytes()
+ num_lines = struct.pack("i", len(lines))
+ structure_data = array.array("i", line_lengths).tobytes()
result = header + coord_data + num_lines + structure_data
@@ -403,17 +412,18 @@ def deserialize_multi_linestring(geom_buffer: GeometryBuffer) -> MultiLineString
return wkt_loads("MULTILINESTRING EMPTY")
return MultiLineString(linestrings)
+
def serialize_polygon(geom: Polygon) -> bytes:
# it may seem odd, but dumping to wkb and parsing proved to be the fastest here
wkb_string = wkb_dumps(geom)
- int_format = ">i" if struct.unpack_from('B', wkb_string) == 0 else "<i"
+ int_format = ">i" if struct.unpack_from("B", wkb_string) == 0 else "<i"
num_rings = struct.unpack_from(int_format, wkb_string, 5)[0]
if num_rings == 0:
return generate_header_bytes(GeometryTypeID.POLYGON, CoordinateType.XY, 0)
- coord_bytes = b''
+ coord_bytes = b""
ring_lengths = []
offset = 9 # 1 byte endianness + 4 byte geom type + 4 byte num rings
bytes_per_coord = geom._ndim * 8
@@ -425,13 +435,13 @@ def serialize_polygon(geom: Polygon) -> bytes:
num_coords += ring_len
offset += 4
- coord_bytes += wkb_string[offset: (offset + bytes_per_coord * ring_len)]
+ coord_bytes += wkb_string[offset : (offset + bytes_per_coord * ring_len)]
offset += bytes_per_coord * ring_len
coord_type = CoordinateType.type_of(geom)
header = generate_header_bytes(GeometryTypeID.POLYGON, coord_type, num_coords)
- structure_data_bytes = array.array('i', [num_rings] + ring_lengths).tobytes()
+ structure_data_bytes = array.array("i", [num_rings] + ring_lengths).tobytes()
return header + coord_bytes + structure_data_bytes
@@ -444,17 +454,37 @@ def deserialize_polygon(geom_buffer: GeometryBuffer) -> Polygon:
def serialize_multi_polygon(geom: MultiPolygon) -> bytes:
coords_for = lambda x: [y for y in list(x)]
- polygons = [[coords_for(polygon.exterior.coords)] + [coords_for(ring.coords) for ring in polygon.interiors] for polygon in list(geom.geoms)]
+ polygons = [
+ [coords_for(polygon.exterior.coords)]
+ + [coords_for(ring.coords) for ring in polygon.interiors]
+ for polygon in list(geom.geoms)
+ ]
coord_type = CoordinateType.type_of(geom)
- structure_data = array.array('i', [val for polygon in polygons for val in [len(polygon)] + [len(ring) for ring in polygon]]).tobytes()
- coords = array.array('d', [component for polygon in polygons for ring in polygon for coord in ring for component in coord]).tobytes()
+ structure_data = array.array(
+ "i",
+ [
+ val
+ for polygon in polygons
+ for val in [len(polygon)] + [len(ring) for ring in polygon]
+ ],
+ ).tobytes()
+ coords = array.array(
+ "d",
+ [
+ component
+ for polygon in polygons
+ for ring in polygon
+ for coord in ring
+ for component in coord
+ ],
+ ).tobytes()
num_coords = len(coords) // CoordinateType.bytes_per_coord(coord_type)
header = generate_header_bytes(GeometryTypeID.MULTIPOLYGON, coord_type, num_coords)
- num_polygons = struct.pack('i', len(polygons))
+ num_polygons = struct.pack("i", len(polygons))
result = header + coords + num_polygons + structure_data
return result
@@ -480,7 +510,9 @@ def serialize_geometry_collection(geom: GeometryCollection) -> bytearray:
return serialize_shapely_1_empty_geom(geom)
geometries = geom.geoms
if not geometries:
- return create_buffer_for_geom(GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, 8, 0)
+ return create_buffer_for_geom(
+ GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, 8, 0
+ )
num_geometries = len(geometries)
total_size = 8
buffers = []
@@ -488,10 +520,12 @@ def serialize_geometry_collection(geom: GeometryCollection) -> bytearray:
buf = serialize(geom)
buffers.append(buf)
total_size += aligned_offset(len(buf))
- buffer = create_buffer_for_geom(GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, total_size, num_geometries)
+ buffer = create_buffer_for_geom(
+ GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, total_size, num_geometries
+ )
offset = 8
for buf in buffers:
- buffer[offset:(offset + len(buf))] = buf
+ buffer[offset : (offset + len(buf))] = buf
offset += aligned_offset(len(buf))
return buffer
diff --git a/python/sedona/utils/jvm.py b/python/sedona/utils/jvm.py
index d66b56596..1b843a001 100644
--- a/python/sedona/utils/jvm.py
+++ b/python/sedona/utils/jvm.py
@@ -28,7 +28,9 @@ class JvmStorageLevel(JvmObject):
@require(["StorageLevel"])
def _create_jvm_instance(self):
return self.jvm.StorageLevel.apply(
- self.storage_level.useDisk, self.storage_level.useMemory,
- self.storage_level.useOffHeap, self.storage_level.deserialized,
- self.storage_level.replication
+ self.storage_level.useDisk,
+ self.storage_level.useMemory,
+ self.storage_level.useOffHeap,
+ self.storage_level.deserialized,
+ self.storage_level.replication,
)
diff --git a/python/sedona/utils/meta.py b/python/sedona/utils/meta.py
index dc8a72343..079b8acb5 100644
--- a/python/sedona/utils/meta.py
+++ b/python/sedona/utils/meta.py
@@ -25,6 +25,7 @@ from typing import Any
try:
from typing import GenericMeta
except ImportError:
+
class GenericMeta(type):
pass
@@ -58,7 +59,9 @@ class MultiMethod:
:param meth:
:return:
"""
- if str(meth).startswith("<classmethod") or str(meth).startswith("<staticmethod"):
+ if str(meth).startswith("<classmethod") or str(meth).startswith(
+ "<staticmethod"
+ ):
sig = inspect.signature(meth.__get__(self))
self._is_static = True
else:
@@ -67,17 +70,17 @@ class MultiMethod:
# Build a type-signature from the method's annotations
types = []
for name, parm in sig.parameters.items():
- if name == 'self':
+ if name == "self":
continue
if name == "args" or name == "kwargs":
raise InvalidParametersException("Can not be used with args and kwargs")
- if name == 'cls':
+ if name == "cls":
continue
if parm.annotation is inspect.Parameter.empty:
raise InvalidParametersException(
- 'Argument {} must be annotated with a type'.format(name)
+ "Argument {} must be annotated with a type".format(name)
)
if parm.default is not inspect.Parameter.empty:
self._methods[tuple(types)] = meth
@@ -107,10 +110,14 @@ class MultiMethod:
methods_which_are_correct = []
for function_methods in methods_shortened_to_args:
- is_instances = all([
- is_subclass_with_typing(from_args, from_definition)
- for from_args, from_definition in zip(types_from_args, function_methods[0])
- ])
+ is_instances = all(
+ [
+ is_subclass_with_typing(from_args, from_definition)
+ for from_args, from_definition in zip(
+ types_from_args, function_methods[0]
+ )
+ ]
+ )
if is_instances:
methods_which_are_correct.append(function_methods[1:])
diff --git a/python/sedona/utils/prep.py b/python/sedona/utils/prep.py
index 3dff253d1..57ea920a7 100644
--- a/python/sedona/utils/prep.py
+++ b/python/sedona/utils/prep.py
@@ -17,12 +17,28 @@
from typing import List
-from shapely.geometry import Point, MultiPoint, Polygon, MultiPolygon, LineString, MultiLineString, GeometryCollection
+from shapely.geometry import (
+ Point,
+ MultiPoint,
+ Polygon,
+ MultiPolygon,
+ LineString,
+ MultiLineString,
+ GeometryCollection,
+)
from shapely.geometry.base import BaseGeometry
def assign_all() -> bool:
- geoms = [Point, MultiPoint, Polygon, MultiPolygon, LineString, MultiLineString, GeometryCollection]
+ geoms = [
+ Point,
+ MultiPoint,
+ Polygon,
+ MultiPolygon,
+ LineString,
+ MultiLineString,
+ GeometryCollection,
+ ]
assign_udt_shapely_objects(geoms=geoms)
assign_user_data_to_shapely_objects(geoms=geoms)
return True
@@ -30,6 +46,7 @@ def assign_all() -> bool:
def assign_udt_shapely_objects(geoms: List[type(BaseGeometry)]) -> bool:
from sedona.sql.types import GeometryType
+
for geom in geoms:
geom.__UDT__ = GeometryType()
return True
diff --git a/python/sedona/utils/spatial_rdd_parser.py b/python/sedona/utils/spatial_rdd_parser.py
index 7f963444b..c18ac9cd8 100644
--- a/python/sedona/utils/spatial_rdd_parser.py
+++ b/python/sedona/utils/spatial_rdd_parser.py
@@ -22,6 +22,7 @@ from typing import List, Any
import attr
from shapely.geometry.base import BaseGeometry
+
try:
from pyspark import CPickleSerializer
except ImportError:
@@ -57,7 +58,7 @@ class GeoData:
return dict(
geom=bytearray([el if el >= 0 else el + 256 for el in geom_bytes]),
- userData=getattr(self, attributes[1])
+ userData=getattr(self, attributes[1]),
)
def __setstate__(self, attributes):
@@ -85,7 +86,9 @@ class GeoData:
__slots__ = ("_geom", "_userData")
def __repr__(self):
- return f"Geometry: {str(self.geom.__class__.__name__)} userData: {self.userData}"
+ return (
+ f"Geometry: {str(self.geom.__class__.__name__)} userData: {self.userData}"
+ )
def __eq__(self, other):
return self.geom == other.geom and self.userData == other.userData
@@ -98,15 +101,15 @@ class GeoData:
class AbstractSpatialRDDParser(ABC):
@classmethod
- def serialize(cls, obj: List[Any], binary_buffer: 'BinaryBuffer') -> bytearray:
+ def serialize(cls, obj: List[Any], binary_buffer: "BinaryBuffer") -> bytearray:
raise NotImplemented()
@classmethod
- def deserialize(cls, bin_parser: 'BinaryParser') -> BaseGeometry:
+ def deserialize(cls, bin_parser: "BinaryParser") -> BaseGeometry:
raise NotImplementedError("Parser has to implement deserialize method")
@classmethod
- def _deserialize_geom(cls, bin_parser: 'BinaryParser') -> GeoData:
+ def _deserialize_geom(cls, bin_parser: "BinaryParser") -> GeoData:
is_circle = bin_parser.read_byte()
return geom_deserializers[is_circle].geometry_from_bytes(bin_parser)
@@ -116,7 +119,7 @@ class SpatialPairRDDParserData(AbstractSpatialRDDParser):
name = "SpatialPairRDDParserData"
@classmethod
- def deserialize(cls, bin_parser: 'BinaryParser'):
+ def deserialize(cls, bin_parser: "BinaryParser"):
left_geom_data = cls._deserialize_geom(bin_parser)
_ = bin_parser.read_int()
@@ -128,7 +131,7 @@ class SpatialPairRDDParserData(AbstractSpatialRDDParser):
return deserialized_data
@classmethod
- def serialize(cls, obj: BaseGeometry, binary_buffer: 'BinaryBuffer'):
+ def serialize(cls, obj: BaseGeometry, binary_buffer: "BinaryBuffer"):
raise NotImplementedError("Currently this operation is not supported")
@@ -137,14 +140,14 @@ class SpatialRDDParserData(AbstractSpatialRDDParser):
name = "SpatialRDDParser"
@classmethod
- def deserialize(cls, bin_parser: 'BinaryParser'):
+ def deserialize(cls, bin_parser: "BinaryParser"):
left_geom_data = cls._deserialize_geom(bin_parser)
_ = bin_parser.read_int()
return left_geom_data
@classmethod
- def serialize(cls, obj: BaseGeometry, binary_buffer: 'BinaryBuffer'):
+ def serialize(cls, obj: BaseGeometry, binary_buffer: "BinaryBuffer"):
raise NotImplementedError("Currently this operation is not supported")
@@ -153,7 +156,7 @@ class SpatialRDDParserDataMultipleRightGeom(AbstractSpatialRDDParser):
name = "SpatialRDDParser"
@classmethod
- def deserialize(cls, bin_parser: 'BinaryParser'):
+ def deserialize(cls, bin_parser: "BinaryParser"):
left_geom_data = cls._deserialize_geom(bin_parser)
geometry_numbers = bin_parser.read_int()
@@ -164,12 +167,14 @@ class SpatialRDDParserDataMultipleRightGeom(AbstractSpatialRDDParser):
right_geom_data = cls._deserialize_geom(bin_parser)
right_geoms.append(right_geom_data)
- deserialized_data = [left_geom_data, right_geoms] if right_geoms else left_geom_data
+ deserialized_data = (
+ [left_geom_data, right_geoms] if right_geoms else left_geom_data
+ )
return deserialized_data
@classmethod
- def serialize(cls, obj: BaseGeometry, binary_buffer: 'BinaryBuffer'):
+ def serialize(cls, obj: BaseGeometry, binary_buffer: "BinaryBuffer"):
raise NotImplementedError("Currently this operation is not supported")
@@ -235,10 +240,11 @@ class CircleGeometryFactory:
@classmethod
def to_bytes(cls, geom: Circle) -> List[int]:
- return struct.pack("b", 1) + struct.pack("d", geom.radius) + dumps(geom.centerGeometry)
+ return (
+ struct.pack("b", 1)
+ + struct.pack("d", geom.radius)
+ + dumps(geom.centerGeometry)
+ )
-geom_deserializers = {
- 1: CircleGeometryFactory,
- 0: GeometryFactory
-}
+geom_deserializers = {1: CircleGeometryFactory, 0: GeometryFactory}
diff --git a/python/setup.py b/python/setup.py
index 7fc01b4e8..fd38f34e2 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -24,48 +24,52 @@ with open("README.md", "r") as fh:
extension_args = {}
-if os.getenv('ENABLE_ASAN'):
+if os.getenv("ENABLE_ASAN"):
extension_args = {
- 'extra_compile_args': ["-fsanitize=address"],
- 'extra_link_args': ["-fsanitize=address"]
+ "extra_compile_args": ["-fsanitize=address"],
+ "extra_link_args": ["-fsanitize=address"],
}
ext_modules = [
- Extension('sedona.utils.geomserde_speedup', sources=[
- 'src/geomserde_speedup_module.c',
- 'src/geomserde.c',
- 'src/geom_buf.c',
- 'src/geos_c_dyn.c'
- ], **extension_args)
+ Extension(
+ "sedona.utils.geomserde_speedup",
+ sources=[
+ "src/geomserde_speedup_module.c",
+ "src/geomserde.c",
+ "src/geom_buf.c",
+ "src/geos_c_dyn.c",
+ ],
+ **extension_args
+ )
]
setup(
- name='apache-sedona',
+ name="apache-sedona",
version=version,
- description='Apache Sedona is a cluster computing system for processing large-scale spatial data',
- url='https://sedona.apache.org',
+ description="Apache Sedona is a cluster computing system for processing large-scale spatial data",
+ url="https://sedona.apache.org",
license="Apache License v2.0",
- author='Apache Sedona',
- author_email='dev@sedona.apache.org',
+ author="Apache Sedona",
+ author_email="dev@sedona.apache.org",
packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
ext_modules=ext_modules,
long_description=long_description,
long_description_content_type="text/markdown",
- python_requires='>=3.6',
- install_requires=['attrs', "shapely>=1.7.0", "rasterio>=1.2.10"],
+ python_requires=">=3.6",
+ install_requires=["attrs", "shapely>=1.7.0", "rasterio>=1.2.10"],
extras_require={
"spark": ["pyspark>=2.3.0"],
"pydeck-map": ["geopandas", "pydeck==0.8.0"],
"kepler-map": ["geopandas", "keplergl==0.3.2"],
- "all": ["pyspark>=2.3.0", "geopandas","pydeck==0.8.0", "keplergl==0.3.2"],
+ "all": ["pyspark>=2.3.0", "geopandas", "pydeck==0.8.0", "keplergl==0.3.2"],
},
project_urls={
- 'Documentation': 'https://sedona.apache.org',
- 'Source code': 'https://github.com/apache/sedona',
- 'Bug Reports': 'https://issues.apache.org/jira/projects/SEDONA'
+ "Documentation": "https://sedona.apache.org",
+ "Source code": "https://github.com/apache/sedona",
+ "Bug Reports": "https://issues.apache.org/jira/projects/SEDONA",
},
classifiers=[
"Programming Language :: Python :: 3",
- "License :: OSI Approved :: Apache Software License"
- ]
+ "License :: OSI Approved :: Apache Software License",
+ ],
)
diff --git a/python/tests/__init__.py b/python/tests/__init__.py
index 96f6a2b86..b63fc7061 100644
--- a/python/tests/__init__.py
+++ b/python/tests/__init__.py
@@ -23,23 +23,43 @@ from tests.tools import tests_resource
mixed_wkb_geometry_input_location = os.path.join(tests_resource, "county_small_wkb.tsv")
mixed_wkt_geometry_input_location = os.path.join(tests_resource, "county_small.tsv")
shape_file_input_location = os.path.join(tests_resource, "shapefiles/dbf")
-shape_file_with_missing_trailing_input_location = os.path.join(tests_resource, "shapefiles/missing")
+shape_file_with_missing_trailing_input_location = os.path.join(
+ tests_resource, "shapefiles/missing"
+)
geojson_input_location = os.path.join(tests_resource, "testPolygon.json")
area_lm_point_input_location = os.path.join(tests_resource, "arealm.csv")
csv_point_input_location = os.path.join(tests_resource, "testpoint.csv")
csv_polygon_input_location = os.path.join(tests_resource, "testenvelope.csv")
-csv_polygon1_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_envelope1.csv")
-csv_polygon2_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_envelope2.csv")
-csv_polygon1_random_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_envelope1_random.csv")
-csv_polygon2_random_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_envelope2_random.csv")
-overlap_polygon_input_location = os.path.join(tests_resource, "testenvelope_overlap.csv")
+csv_polygon1_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_envelope1.csv"
+)
+csv_polygon2_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_envelope2.csv"
+)
+csv_polygon1_random_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_envelope1_random.csv"
+)
+csv_polygon2_random_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_envelope2_random.csv"
+)
+overlap_polygon_input_location = os.path.join(
+ tests_resource, "testenvelope_overlap.csv"
+)
union_polygon_input_location = os.path.join(tests_resource, "testunion.csv")
-csv_point1_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_point1.csv")
-csv_point2_input_location = os.path.join(tests_resource, "equalitycheckfiles/testequals_point2.csv")
+csv_point1_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_point1.csv"
+)
+csv_point2_input_location = os.path.join(
+ tests_resource, "equalitycheckfiles/testequals_point2.csv"
+)
geojson_id_input_location = os.path.join(tests_resource, "testContainsId.json")
geoparquet_input_location = os.path.join(tests_resource, "geoparquet/example1.parquet")
plain_parquet_input_location = os.path.join(tests_resource, "geoparquet/plain.parquet")
-legacy_parquet_input_location = os.path.join(tests_resource, "parquet/legacy-parquet-nested-columns.snappy.parquet")
+legacy_parquet_input_location = os.path.join(
+ tests_resource, "parquet/legacy-parquet-nested-columns.snappy.parquet"
+)
google_buildings_input_location = os.path.join(tests_resource, "813_buildings_test.csv")
chicago_crimes_input_location = os.path.join(tests_resource, "Chicago_Crimes.csv")
-world_map_raster_input_location = os.path.join(tests_resource, "raster/raster_with_no_data/test5.tiff")
+world_map_raster_input_location = os.path.join(
+ tests_resource, "raster/raster_with_no_data/test5.tiff"
+)
diff --git a/python/tests/core/test_avoiding_python_jvm_serde_df.py b/python/tests/core/test_avoiding_python_jvm_serde_df.py
index d86a965f1..4811f9f51 100644
--- a/python/tests/core/test_avoiding_python_jvm_serde_df.py
+++ b/python/tests/core/test_avoiding_python_jvm_serde_df.py
@@ -36,26 +36,50 @@ areas_csv_path = os.path.join(tests_resource, "small/areas.csv")
class TestOmitPythonJvmSerdeToDf(TestBase):
- expected_pois_within_areas_ids = [['4', '4'], ['1', '6'], ['2', '1'], ['3', '3'], ['3', '7']]
+ expected_pois_within_areas_ids = [
+ ["4", "4"],
+ ["1", "6"],
+ ["2", "1"],
+ ["3", "3"],
+ ["3", "7"],
+ ]
def test_spatial_join_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
- areas_polygon_rdd = WktReader.readToGeometryRDD(self.sc, areas_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
+ areas_polygon_rdd = WktReader.readToGeometryRDD(
+ self.sc, areas_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
areas_polygon_rdd.analyze()
poi_point_rdd.spatialPartitioning(GridType.QUADTREE)
areas_polygon_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
- jvm_sedona_rdd = JoinQueryRaw.spatialJoin(poi_point_rdd, areas_polygon_rdd, JoinParams(considerBoundaryIntersection=True))
- sedona_df = Adapter.toDf(jvm_sedona_rdd, ["area_id", "area_name"], ["poi_id", "poi_name"], self.spark)
+ jvm_sedona_rdd = JoinQueryRaw.spatialJoin(
+ poi_point_rdd,
+ areas_polygon_rdd,
+ JoinParams(considerBoundaryIntersection=True),
+ )
+ sedona_df = Adapter.toDf(
+ jvm_sedona_rdd, ["area_id", "area_name"], ["poi_id", "poi_name"], self.spark
+ )
assert sedona_df.count() == 5
- assert sedona_df.columns == ["leftgeometry", "area_id", "area_name", "rightgeometry",
- "poi_id", "poi_name"]
+ assert sedona_df.columns == [
+ "leftgeometry",
+ "area_id",
+ "area_name",
+ "rightgeometry",
+ "poi_id",
+ "poi_name",
+ ]
def test_distance_join_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
circle_rdd = CircleRDD(poi_point_rdd, 2.0)
circle_rdd.analyze()
@@ -64,12 +88,14 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
poi_point_rdd.spatialPartitioning(GridType.QUADTREE)
circle_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
- jvm_sedona_rdd = JoinQueryRaw.DistanceJoinQueryFlat(poi_point_rdd, circle_rdd, False, True)
+ jvm_sedona_rdd = JoinQueryRaw.DistanceJoinQueryFlat(
+ poi_point_rdd, circle_rdd, False, True
+ )
df_sedona_rdd = Adapter.toDf(
jvm_sedona_rdd,
["poi_from_id", "poi_from_name"],
["poi_to_id", "poi_to_name"],
- self.spark
+ self.spark,
)
assert df_sedona_rdd.count() == 10
@@ -79,12 +105,16 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
"poi_from_name",
"rightgeometry",
"poi_to_id",
- "poi_to_name"
+ "poi_to_name",
]
def test_spatial_join_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
- areas_polygon_rdd = WktReader.readToGeometryRDD(self.sc, areas_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
+ areas_polygon_rdd = WktReader.readToGeometryRDD(
+ self.sc, areas_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
areas_polygon_rdd.analyze()
@@ -92,24 +122,29 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
areas_polygon_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
jvm_sedona_rdd = JoinQueryRaw.SpatialJoinQueryFlat(
- poi_point_rdd, areas_polygon_rdd, False, True)
+ poi_point_rdd, areas_polygon_rdd, False, True
+ )
- pois_within_areas_with_default_column_names = Adapter.toDf(jvm_sedona_rdd, self.spark)
+ pois_within_areas_with_default_column_names = Adapter.toDf(
+ jvm_sedona_rdd, self.spark
+ )
assert pois_within_areas_with_default_column_names.count() == 5
pois_within_areas_with_passed_column_names = Adapter.toDf(
- jvm_sedona_rdd,
- ["area_id", "area_name"],
- ["poi_id", "poi_name"],
- self.spark
+ jvm_sedona_rdd, ["area_id", "area_name"], ["poi_id", "poi_name"], self.spark
)
assert pois_within_areas_with_passed_column_names.count() == 5
- assert pois_within_areas_with_passed_column_names.columns == ["leftgeometry", "area_id", "area_name",
- "rightgeometry",
- "poi_id", "poi_name"]
+ assert pois_within_areas_with_passed_column_names.columns == [
+ "leftgeometry",
+ "area_id",
+ "area_name",
+ "rightgeometry",
+ "poi_id",
+ "poi_name",
+ ]
assert pois_within_areas_with_default_column_names.schema == StructType(
[
@@ -118,36 +153,42 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
]
)
- left_geometries_raw = pois_within_areas_with_default_column_names. \
- selectExpr("ST_AsText(leftgeometry)"). \
- collect()
+ left_geometries_raw = pois_within_areas_with_default_column_names.selectExpr(
+ "ST_AsText(leftgeometry)"
+ ).collect()
left_geometries = self.__row_to_list(left_geometries_raw)
- right_geometries_raw = pois_within_areas_with_default_column_names. \
- selectExpr("ST_AsText(rightgeometry)"). \
- collect()
+ right_geometries_raw = pois_within_areas_with_default_column_names.selectExpr(
+ "ST_AsText(rightgeometry)"
+ ).collect()
right_geometries = self.__row_to_list(right_geometries_raw)
# Ignore the ordering of these
- assert set(geom[0] for geom in left_geometries) == set([
- 'POLYGON ((0 4, -3 3, -8 6, -6 8, -2 9, 0 4))',
- 'POLYGON ((10 3, 10 6, 14 6, 14 3, 10 3))',
- 'POLYGON ((2 2, 2 4, 3 5, 7 5, 9 3, 8 1, 4 1, 2 2))',
- 'POLYGON ((-1 -1, -1 -3, -2 -5, -6 -8, -5 -2, -3 -2, -1 -1))',
- 'POLYGON ((-1 -1, -1 -3, -2 -5, -6 -8, -5 -2, -3 -2, -1 -1))'
- ])
- assert set(geom[0] for geom in right_geometries) == set([
- 'POINT (-3 5)',
- 'POINT (11 5)',
- 'POINT (4 3)',
- 'POINT (-1 -1)',
- 'POINT (-4 -5)'
- ])
+ assert set(geom[0] for geom in left_geometries) == set(
+ [
+ "POLYGON ((0 4, -3 3, -8 6, -6 8, -2 9, 0 4))",
+ "POLYGON ((10 3, 10 6, 14 6, 14 3, 10 3))",
+ "POLYGON ((2 2, 2 4, 3 5, 7 5, 9 3, 8 1, 4 1, 2 2))",
+ "POLYGON ((-1 -1, -1 -3, -2 -5, -6 -8, -5 -2, -3 -2, -1 -1))",
+ "POLYGON ((-1 -1, -1 -3, -2 -5, -6 -8, -5 -2, -3 -2, -1 -1))",
+ ]
+ )
+ assert set(geom[0] for geom in right_geometries) == set(
+ [
+ "POINT (-3 5)",
+ "POINT (11 5)",
+ "POINT (4 3)",
+ "POINT (-1 -1)",
+ "POINT (-4 -5)",
+ ]
+ )
def test_range_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
@@ -164,15 +205,18 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
df_without_column_names = Adapter.toDf(result, self.spark)
- raw_geometries = self.__row_to_list(
- df_without_column_names.collect()
- )
+ raw_geometries = self.__row_to_list(df_without_column_names.collect())
assert [point[0].wkt for point in raw_geometries] == [
- 'POINT (9 8)', 'POINT (4 3)', 'POINT (12 1)', 'POINT (11 5)'
+ "POINT (9 8)",
+ "POINT (4 3)",
+ "POINT (12 1)",
+ "POINT (11 5)",
]
assert df_without_column_names.count() == 4
- assert df_without_column_names.schema == StructType([StructField("geometry", GeometryType())])
+ assert df_without_column_names.schema == StructType(
+ [StructField("geometry", GeometryType())]
+ )
df = Adapter.toDf(result, self.spark, ["poi_id", "poi_name"])
diff --git a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
index bfbd1de1d..382934259 100644
--- a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
+++ b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py
@@ -33,23 +33,39 @@ areas_csv_path = os.path.join(tests_resource, "small/areas.csv")
class TestOmitPythonJvmSerdeToRDD(TestBase):
- expected_pois_within_areas_ids = [['4', '4'], ['1', '6'], ['2', '1'], ['3', '3'], ['3', '7']]
+ expected_pois_within_areas_ids = [
+ ["4", "4"],
+ ["1", "6"],
+ ["2", "1"],
+ ["3", "3"],
+ ["3", "7"],
+ ]
def test_spatial_join_to_spatial_rdd(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
- areas_polygon_rdd = WktReader.readToGeometryRDD(self.sc, areas_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
+ areas_polygon_rdd = WktReader.readToGeometryRDD(
+ self.sc, areas_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
areas_polygon_rdd.analyze()
poi_point_rdd.spatialPartitioning(GridType.QUADTREE)
areas_polygon_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
- jvm_sedona_rdd = JoinQueryRaw.spatialJoin(poi_point_rdd, areas_polygon_rdd, JoinParams(considerBoundaryIntersection=True))
+ jvm_sedona_rdd = JoinQueryRaw.spatialJoin(
+ poi_point_rdd,
+ areas_polygon_rdd,
+ JoinParams(considerBoundaryIntersection=True),
+ )
sedona_rdd = jvm_sedona_rdd.to_rdd().collect()
assert sedona_rdd.__len__() == 5
def test_distance_join_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
circle_rdd = CircleRDD(poi_point_rdd, 2.0)
circle_rdd.analyze()
@@ -58,13 +74,19 @@ class TestOmitPythonJvmSerdeToRDD(TestBase):
poi_point_rdd.spatialPartitioning(GridType.QUADTREE)
circle_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
- jvm_sedona_rdd = JoinQueryRaw.DistanceJoinQueryFlat(poi_point_rdd, circle_rdd, False, True)
+ jvm_sedona_rdd = JoinQueryRaw.DistanceJoinQueryFlat(
+ poi_point_rdd, circle_rdd, False, True
+ )
assert jvm_sedona_rdd.to_rdd().collect().__len__() == 10
def test_spatial_join_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
- areas_polygon_rdd = WktReader.readToGeometryRDD(self.sc, areas_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
+ areas_polygon_rdd = WktReader.readToGeometryRDD(
+ self.sc, areas_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
areas_polygon_rdd.analyze()
@@ -72,12 +94,15 @@ class TestOmitPythonJvmSerdeToRDD(TestBase):
areas_polygon_rdd.spatialPartitioning(poi_point_rdd.getPartitioner())
jvm_sedona_rdd = JoinQueryRaw.SpatialJoinQueryFlat(
- poi_point_rdd, areas_polygon_rdd, False, True)
+ poi_point_rdd, areas_polygon_rdd, False, True
+ )
assert jvm_sedona_rdd.to_rdd().collect().__len__() == 5
def test_range_query_flat_to_df(self):
- poi_point_rdd = WktReader.readToGeometryRDD(self.sc, bank_csv_path, 1, False, False)
+ poi_point_rdd = WktReader.readToGeometryRDD(
+ self.sc, bank_csv_path, 1, False, False
+ )
poi_point_rdd.analyze()
diff --git a/python/tests/core/test_core_geom_primitives.py b/python/tests/core/test_core_geom_primitives.py
index 6794d1f90..3fa20a369 100644
--- a/python/tests/core/test_core_geom_primitives.py
+++ b/python/tests/core/test_core_geom_primitives.py
@@ -25,4 +25,6 @@ class TestGeomPrimitives(TestBase):
envelope = Envelope(0.0, 5.0, 0.0, 5.0)
jvm_instance = envelope.create_jvm_instance(self.spark.sparkContext._jvm)
envelope_area = jvm_instance.getArea()
- assert envelope_area == 25.0, f"Expected area to be equal 25 but {envelope_area} was found"
+ assert (
+ envelope_area == 25.0
+ ), f"Expected area to be equal 25 but {envelope_area} was found"
diff --git a/python/tests/core/test_core_rdd.py b/python/tests/core/test_core_rdd.py
index 0473164f1..769c0a1eb 100644
--- a/python/tests/core/test_core_rdd.py
+++ b/python/tests/core/test_core_rdd.py
@@ -30,13 +30,7 @@ counties_path = os.path.join(tests_resource, "county_small.tsv")
class TestSpatialRDD(TestBase):
def test_creating_point_rdd(self):
- point_rdd = PointRDD(
- self.spark._sc,
- point_path,
- 4,
- FileDataSplitter.WKT,
- True
- )
+ point_rdd = PointRDD(self.spark._sc, point_path, 4, FileDataSplitter.WKT, True)
point_rdd.analyze()
cnt = point_rdd.countWithoutDuplicates()
@@ -44,12 +38,7 @@ class TestSpatialRDD(TestBase):
def test_creating_polygon_rdd(self):
polygon_rdd = PolygonRDD(
- self.spark._sc,
- counties_path,
- 0,
- 1,
- FileDataSplitter.WKT,
- True
+ self.spark._sc, counties_path, 0, 1, FileDataSplitter.WKT, True
)
polygon_rdd.analyze()
diff --git a/python/tests/core/test_rdd.py b/python/tests/core/test_rdd.py
index 201d44292..a002c7db2 100644
--- a/python/tests/core/test_rdd.py
+++ b/python/tests/core/test_rdd.py
@@ -27,8 +27,13 @@ from sedona.core.spatialOperator import RangeQuery, KNNQuery, JoinQuery
from sedona.core.spatialOperator.join_params import JoinParams
import os
-from tests.properties.polygon_properties import polygon_rdd_input_location, polygon_rdd_start_offset, polygon_rdd_end_offset, \
- polygon_rdd_splitter, polygon_rdd_index_type
+from tests.properties.polygon_properties import (
+ polygon_rdd_input_location,
+ polygon_rdd_start_offset,
+ polygon_rdd_end_offset,
+ polygon_rdd_splitter,
+ polygon_rdd_index_type,
+)
from tests.test_base import TestBase
from tests.tools import tests_resource
@@ -56,7 +61,7 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd_copy = PointRDD()
object_rdd_copy.rawJvmSpatialRDD = object_rdd.rawJvmSpatialRDD
@@ -68,7 +73,8 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False)
+ carryInputData=False,
+ )
for i in range(each_query_loop_times):
result_size = RangeQuery.SpatialRangeQuery(
@@ -82,12 +88,13 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd.buildIndex(point_rdd_index_type, False)
for i in range(each_query_loop_times):
result_size = RangeQuery.SpatialRangeQuery(
- object_rdd, range_query_window, False, True).count
+ object_rdd, range_query_window, False, True
+ ).count
def test_knn_query(self):
object_rdd = PointRDD(
@@ -95,7 +102,7 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
for i in range(each_query_loop_times):
result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, False)
@@ -106,10 +113,10 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd.buildIndex(point_rdd_index_type, False)
- for i in range(each_query_loop_times):
+ for i in range(each_query_loop_times):
result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, True)
def test_spaltial_join(self):
@@ -119,7 +126,7 @@ class TestSpatialRDD(TestBase):
polygon_rdd_start_offset,
polygon_rdd_end_offset,
polygon_rdd_splitter,
- True
+ True,
)
object_rdd = PointRDD(
@@ -127,7 +134,7 @@ class TestSpatialRDD(TestBase):
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd.analyze()
object_rdd.spatialPartitioning(join_query_partitionin_type)
@@ -135,7 +142,8 @@ class TestSpatialRDD(TestBase):
for x in range(each_query_loop_times):
result_size = JoinQuery.SpatialJoinQuery(
- object_rdd, query_window_rdd, False, True).count
+ object_rdd, query_window_rdd, False, True
+ ).count
def test_spatial_join_using_index(self):
query_window = PolygonRDD(
@@ -144,14 +152,14 @@ class TestSpatialRDD(TestBase):
polygon_rdd_start_offset,
polygon_rdd_end_offset,
polygon_rdd_splitter,
- True
+ True,
)
object_rdd = PointRDD(
sparkContext=self.sc,
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd.analyze()
object_rdd.spatialPartitioning(join_query_partitionin_type)
@@ -161,7 +169,8 @@ class TestSpatialRDD(TestBase):
for i in range(each_query_loop_times):
result_size = JoinQuery.SpatialJoinQuery(
- object_rdd, query_window, True, False).count()
+ object_rdd, query_window, True, False
+ ).count()
def test_spatial_join_using_index_on_polygons(self):
query_window = PolygonRDD(
@@ -170,14 +179,14 @@ class TestSpatialRDD(TestBase):
polygon_rdd_start_offset,
polygon_rdd_end_offset,
polygon_rdd_splitter,
- True
+ True,
)
object_rdd = PointRDD(
sparkContext=self.sc,
InputLocation=point_rdd_input_location,
Offset=point_rdd_offset,
splitter=point_rdd_splitter,
- carryInputData=False
+ carryInputData=False,
)
object_rdd.analyze()
object_rdd.spatialPartitioning(join_query_partitionin_type)
@@ -187,10 +196,7 @@ class TestSpatialRDD(TestBase):
for i in range(each_query_loop_times):
result_size = JoinQuery.SpatialJoinQuery(
- object_rdd,
- query_window,
- True,
- False
+ object_rdd, query_window, True, False
).count()
def test_spatial_join_query_using_index_on_polygons(self):
@@ -200,14 +206,14 @@ class TestSpatialRDD(TestBase):
polygon_rdd_start_offset,
polygon_rdd_end_offset,
polygon_rdd_splitter,
- True
+ True,
)
object_rdd = PointRDD(
sparkContext=self.sc,
InputLocation=point_rdd_input_location,
... 12588 lines suppressed ...