You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2021/10/09 03:24:03 UTC
[incubator-sedona] branch master updated: [DOCS] Update Binder
notebook examples to Sedona 1.1.0
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new cb2c7d1 [DOCS] Update Binder notebook examples to Sedona 1.1.0
cb2c7d1 is described below
commit cb2c7d1e5226a40ae1d78e6013d0def3bd169750
Author: Jia Yu <ji...@apache.org>
AuthorDate: Fri Oct 8 20:23:35 2021 -0700
[DOCS] Update Binder notebook examples to Sedona 1.1.0
---
binder/ApacheSedonaCore.ipynb | 324 ++++++++++-----------
...donaRaster_1.ipynb => ApacheSedonaRaster.ipynb} | 215 ++++++++++----
binder/ApacheSedonaSQL.ipynb | 274 ++++++++++++-----
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 133 +++++++--
binder/Pipfile | 4 +-
binder/postBuild | 6 +-
binder/start | 2 +-
7 files changed, 627 insertions(+), 331 deletions(-)
diff --git a/binder/ApacheSedonaCore.ipynb b/binder/ApacheSedonaCore.ipynb
index e311b99..bc829c7 100644
--- a/binder/ApacheSedonaCore.ipynb
+++ b/binder/ApacheSedonaCore.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -43,7 +43,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -53,13 +53,13 @@
" appName(\"Sedona App\").\\\n",
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName) .\\\n",
- " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.0.0-incubating,org.datasyslab:geotools-wrapper:geotools-24.0\") .\\\n",
+ " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2-RC1\") .\\\n",
" getOrCreate()"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -68,7 +68,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
@@ -77,7 +77,7 @@
"True"
]
},
- "execution_count": 4,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -88,7 +88,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
@@ -126,7 +126,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -135,7 +135,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
@@ -144,7 +144,7 @@
"3000"
]
},
- "execution_count": 7,
+ "execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
@@ -156,7 +156,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -168,7 +168,7 @@
"Envelope(-173.120769, -84.965961, 30.244859, 71.355134)"
]
},
- "execution_count": 8,
+ "execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
@@ -181,7 +181,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -190,7 +190,7 @@
"True"
]
},
- "execution_count": 9,
+ "execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
@@ -202,7 +202,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -214,7 +214,7 @@
"Envelope(-173.120769, -84.965961, 30.244859, 71.355134)"
]
},
- "execution_count": 10,
+ "execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
@@ -227,7 +227,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -236,7 +236,7 @@
"2996"
]
},
- "execution_count": 11,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -248,7 +248,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
@@ -257,7 +257,7 @@
"'epsg:4326'"
]
},
- "execution_count": 12,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -269,7 +269,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -278,7 +278,7 @@
"'epsg:4326'"
]
},
- "execution_count": 13,
+ "execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
@@ -290,7 +290,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 19,
"metadata": {},
"outputs": [
{
@@ -299,7 +299,7 @@
"True"
]
},
- "execution_count": 14,
+ "execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
@@ -329,7 +329,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
@@ -338,7 +338,7 @@
"[Geometry: Point userData: testattribute0\ttestattribute1\ttestattribute2]"
]
},
- "execution_count": 15,
+ "execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
@@ -350,7 +350,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -363,7 +363,7 @@
" Geometry: Point userData: testattribute0\ttestattribute1\ttestattribute2]"
]
},
- "execution_count": 16,
+ "execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
@@ -375,7 +375,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -388,7 +388,7 @@
" 110.97122518072091]"
]
},
- "execution_count": 17,
+ "execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
@@ -421,7 +421,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -430,7 +430,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
@@ -441,7 +441,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 25,
"metadata": {},
"outputs": [
{
@@ -520,7 +520,7 @@
"4 POINT (-88.32399 32.95067) testattribute0 testattribute1 testattribute2"
]
},
- "execution_count": 20,
+ "execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
@@ -538,7 +538,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
@@ -547,7 +547,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
@@ -560,7 +560,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": 28,
"metadata": {},
"outputs": [
{
@@ -587,7 +587,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
@@ -666,7 +666,7 @@
"4 testattribute0 testattribute1 testattribute2 POINT (-88.32399 32.95067)"
]
},
- "execution_count": 24,
+ "execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
@@ -684,7 +684,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
@@ -700,7 +700,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
@@ -709,7 +709,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 32,
"metadata": {},
"outputs": [
{
@@ -788,7 +788,7 @@
"4 POINT (-88.32399 32.95067) testattribute0 testattribute1 testattribute2"
]
},
- "execution_count": 27,
+ "execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
@@ -818,7 +818,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
@@ -830,7 +830,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 34,
"metadata": {},
"outputs": [
{
@@ -839,7 +839,7 @@
"True"
]
},
- "execution_count": 29,
+ "execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
@@ -867,7 +867,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 35,
"metadata": {},
"outputs": [
{
@@ -876,7 +876,7 @@
"True"
]
},
- "execution_count": 30,
+ "execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
@@ -901,7 +901,7 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
@@ -949,7 +949,7 @@
},
{
"cell_type": "code",
- "execution_count": 32,
+ "execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
@@ -972,16 +972,16 @@
},
{
"cell_type": "code",
- "execution_count": 33,
+ "execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "MapPartitionsRDD[63] at map at FlatPairRddConverter.scala:30"
+ "MapPartitionsRDD[68] at map at FlatPairRddConverter.scala:30"
]
},
- "execution_count": 33,
+ "execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
@@ -992,7 +992,7 @@
},
{
"cell_type": "code",
- "execution_count": 34,
+ "execution_count": 39,
"metadata": {},
"outputs": [
{
@@ -1002,7 +1002,7 @@
" [Geometry: Polygon userData: , Geometry: Point userData: ]]"
]
},
- "execution_count": 34,
+ "execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
@@ -1013,7 +1013,7 @@
},
{
"cell_type": "code",
- "execution_count": 35,
+ "execution_count": 40,
"metadata": {},
"outputs": [
{
@@ -1024,7 +1024,7 @@
" [Geometry: Polygon userData: , Geometry: Point userData: ]]"
]
},
- "execution_count": 35,
+ "execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
@@ -1035,7 +1035,7 @@
},
{
"cell_type": "code",
- "execution_count": 36,
+ "execution_count": 41,
"metadata": {},
"outputs": [
{
@@ -1044,7 +1044,7 @@
"[0.0, 0.0, 0.0, 0.0, 0.0]"
]
},
- "execution_count": 36,
+ "execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
@@ -1056,20 +1056,20 @@
},
{
"cell_type": "code",
- "execution_count": 37,
+ "execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "[0.010747596697999453,\n",
- " 0.010747596697999453,\n",
- " 0.010747596697999453,\n",
- " 0.010747596697999453,\n",
- " 0.026651558685001447]"
+ "[0.05189354027999942,\n",
+ " 0.051572544132000575,\n",
+ " 0.05189354027999942,\n",
+ " 0.05189354027999942,\n",
+ " 0.020876428870001032]"
]
},
- "execution_count": 37,
+ "execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
@@ -1081,7 +1081,7 @@
},
{
"cell_type": "code",
- "execution_count": 38,
+ "execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
@@ -1090,7 +1090,7 @@
},
{
"cell_type": "code",
- "execution_count": 39,
+ "execution_count": 44,
"metadata": {},
"outputs": [],
"source": [
@@ -1104,7 +1104,7 @@
},
{
"cell_type": "code",
- "execution_count": 40,
+ "execution_count": 45,
"metadata": {},
"outputs": [
{
@@ -1114,11 +1114,11 @@
"+--------------------+--------------------+\n",
"| geom_left| geom_right|\n",
"+--------------------+--------------------+\n",
- "|POLYGON ((-86.749...|POINT (-86.736302...|\n",
- "|POLYGON ((-86.749...|POINT (-86.735506...|\n",
- "|POLYGON ((-86.749...|POINT (-86.68645 ...|\n",
- "|POLYGON ((-86.749...|POINT (-86.675405...|\n",
- "|POLYGON ((-87.229...|POINT (-87.105455...|\n",
+ "|POLYGON ((-87.092...|POINT (-86.94719 ...|\n",
+ "|POLYGON ((-87.082...|POINT (-86.865782...|\n",
+ "|POLYGON ((-87.092...|POINT (-86.779523...|\n",
+ "|POLYGON ((-87.092...|POINT (-86.865782...|\n",
+ "|POLYGON ((-86.816...|POINT (-86.779523...|\n",
"+--------------------+--------------------+\n",
"only showing top 5 rows\n",
"\n"
@@ -1133,7 +1133,7 @@
},
{
"cell_type": "code",
- "execution_count": 41,
+ "execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
@@ -1142,7 +1142,7 @@
},
{
"cell_type": "code",
- "execution_count": 42,
+ "execution_count": 47,
"metadata": {},
"outputs": [
{
@@ -1169,7 +1169,7 @@
},
{
"cell_type": "code",
- "execution_count": 43,
+ "execution_count": 48,
"metadata": {},
"outputs": [
{
@@ -1179,11 +1179,11 @@
"+--------------------+-----+--------------------+-----+\n",
"| geom_1|attr1| geom_2|attr2|\n",
"+--------------------+-----+--------------------+-----+\n",
- "|POLYGON ((-86.749...| |POINT (-86.736302...| |\n",
- "|POLYGON ((-86.749...| |POINT (-86.735506...| |\n",
- "|POLYGON ((-86.749...| |POINT (-86.68645 ...| |\n",
- "|POLYGON ((-86.749...| |POINT (-86.675405...| |\n",
- "|POLYGON ((-87.229...| |POINT (-87.105455...| |\n",
+ "|POLYGON ((-87.092...| |POINT (-86.94719 ...| |\n",
+ "|POLYGON ((-87.082...| |POINT (-86.865782...| |\n",
+ "|POLYGON ((-87.092...| |POINT (-86.779523...| |\n",
+ "|POLYGON ((-87.092...| |POINT (-86.865782...| |\n",
+ "|POLYGON ((-86.816...| |POINT (-86.779523...| |\n",
"+--------------------+-----+--------------------+-----+\n",
"only showing top 5 rows\n",
"\n"
@@ -1203,7 +1203,7 @@
},
{
"cell_type": "code",
- "execution_count": 44,
+ "execution_count": 49,
"metadata": {},
"outputs": [
{
@@ -1240,7 +1240,7 @@
},
{
"cell_type": "code",
- "execution_count": 45,
+ "execution_count": 50,
"metadata": {},
"outputs": [],
"source": [
@@ -1250,7 +1250,7 @@
},
{
"cell_type": "code",
- "execution_count": 46,
+ "execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
@@ -1259,7 +1259,7 @@
},
{
"cell_type": "code",
- "execution_count": 47,
+ "execution_count": 52,
"metadata": {},
"outputs": [],
"source": [
@@ -1269,7 +1269,7 @@
},
{
"cell_type": "code",
- "execution_count": 48,
+ "execution_count": 53,
"metadata": {},
"outputs": [],
"source": [
@@ -1281,7 +1281,7 @@
},
{
"cell_type": "code",
- "execution_count": 49,
+ "execution_count": 54,
"metadata": {},
"outputs": [
{
@@ -1291,16 +1291,16 @@
"+--------------------+----------------+\n",
"| geometry|number_of_points|\n",
"+--------------------+----------------+\n",
- "|POLYGON ((-86.860...| 12|\n",
- "|POLYGON ((-87.082...| 12|\n",
"|POLYGON ((-86.749...| 4|\n",
- "|POLYGON ((-87.285...| 26|\n",
- "|POLYGON ((-87.105...| 15|\n",
- "|POLYGON ((-87.092...| 5|\n",
+ "|POLYGON ((-87.082...| 12|\n",
"|POLYGON ((-86.697...| 1|\n",
+ "|POLYGON ((-87.105...| 15|\n",
+ "|POLYGON ((-87.114...| 15|\n",
"|POLYGON ((-87.229...| 7|\n",
+ "|POLYGON ((-86.860...| 12|\n",
"|POLYGON ((-86.816...| 6|\n",
- "|POLYGON ((-87.114...| 15|\n",
+ "|POLYGON ((-87.092...| 5|\n",
+ "|POLYGON ((-87.285...| 26|\n",
"+--------------------+----------------+\n",
"\n"
]
@@ -1333,7 +1333,7 @@
},
{
"cell_type": "code",
- "execution_count": 50,
+ "execution_count": 55,
"metadata": {},
"outputs": [],
"source": [
@@ -1342,7 +1342,7 @@
},
{
"cell_type": "code",
- "execution_count": 51,
+ "execution_count": 56,
"metadata": {},
"outputs": [],
"source": [
@@ -1351,7 +1351,7 @@
},
{
"cell_type": "code",
- "execution_count": 52,
+ "execution_count": 57,
"metadata": {},
"outputs": [
{
@@ -1364,7 +1364,7 @@
" Geometry: Point userData: ]"
]
},
- "execution_count": 52,
+ "execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
@@ -1382,7 +1382,7 @@
},
{
"cell_type": "code",
- "execution_count": 53,
+ "execution_count": 58,
"metadata": {},
"outputs": [],
"source": [
@@ -1396,7 +1396,7 @@
},
{
"cell_type": "code",
- "execution_count": 54,
+ "execution_count": 59,
"metadata": {},
"outputs": [
{
@@ -1409,7 +1409,7 @@
" Geometry: Polygon userData: ]"
]
},
- "execution_count": 54,
+ "execution_count": 59,
"metadata": {},
"output_type": "execute_result"
}
@@ -1420,16 +1420,16 @@
},
{
"cell_type": "code",
- "execution_count": 55,
+ "execution_count": 60,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "'POLYGON ((-84.031975 34.043824, -84.031975 34.131247, -83.959903 34.131247, -83.959903 34.043824, -84.031975 34.043824))'"
+ "'POLYGON ((-83.993559 34.087259, -83.993559 34.131247, -83.959903 34.131247, -83.959903 34.087259, -83.993559 34.087259))'"
]
},
- "execution_count": 55,
+ "execution_count": 60,
"metadata": {},
"output_type": "execute_result"
}
@@ -1459,7 +1459,7 @@
},
{
"cell_type": "code",
- "execution_count": 56,
+ "execution_count": 61,
"metadata": {},
"outputs": [],
"source": [
@@ -1470,16 +1470,16 @@
},
{
"cell_type": "code",
- "execution_count": 57,
+ "execution_count": 62,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "MapPartitionsRDD[126] at map at GeometryRddConverter.scala:30"
+ "MapPartitionsRDD[131] at map at GeometryRddConverter.scala:30"
]
},
- "execution_count": 57,
+ "execution_count": 62,
"metadata": {},
"output_type": "execute_result"
}
@@ -1490,7 +1490,7 @@
},
{
"cell_type": "code",
- "execution_count": 58,
+ "execution_count": 63,
"metadata": {},
"outputs": [
{
@@ -1504,7 +1504,7 @@
" Geometry: LineString userData: ]"
]
},
- "execution_count": 58,
+ "execution_count": 63,
"metadata": {},
"output_type": "execute_result"
}
@@ -1515,7 +1515,7 @@
},
{
"cell_type": "code",
- "execution_count": 59,
+ "execution_count": 64,
"metadata": {},
"outputs": [],
"source": [
@@ -1524,7 +1524,7 @@
},
{
"cell_type": "code",
- "execution_count": 60,
+ "execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
@@ -1533,7 +1533,7 @@
},
{
"cell_type": "code",
- "execution_count": 61,
+ "execution_count": 66,
"metadata": {},
"outputs": [
{
@@ -1582,7 +1582,7 @@
},
{
"cell_type": "code",
- "execution_count": 62,
+ "execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
@@ -1591,7 +1591,7 @@
},
{
"cell_type": "code",
- "execution_count": 63,
+ "execution_count": 68,
"metadata": {},
"outputs": [],
"source": [
@@ -1600,16 +1600,16 @@
},
{
"cell_type": "code",
- "execution_count": 64,
+ "execution_count": 69,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f9176a4d828>"
+ "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x1203b87c0>"
]
},
- "execution_count": 64,
+ "execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
@@ -1620,7 +1620,7 @@
},
{
"cell_type": "code",
- "execution_count": 65,
+ "execution_count": 70,
"metadata": {},
"outputs": [
{
@@ -1647,7 +1647,7 @@
},
{
"cell_type": "code",
- "execution_count": 66,
+ "execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
@@ -1665,7 +1665,7 @@
},
{
"cell_type": "code",
- "execution_count": 67,
+ "execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
@@ -1674,16 +1674,16 @@
},
{
"cell_type": "code",
- "execution_count": 68,
+ "execution_count": 73,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f9176a4e630>"
+ "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x12038c940>"
]
},
- "execution_count": 68,
+ "execution_count": 73,
"metadata": {},
"output_type": "execute_result"
}
@@ -1694,7 +1694,7 @@
},
{
"cell_type": "code",
- "execution_count": 69,
+ "execution_count": 74,
"metadata": {},
"outputs": [
{
@@ -1721,7 +1721,7 @@
},
{
"cell_type": "code",
- "execution_count": 70,
+ "execution_count": 75,
"metadata": {},
"outputs": [],
"source": [
@@ -1730,7 +1730,7 @@
},
{
"cell_type": "code",
- "execution_count": 71,
+ "execution_count": 76,
"metadata": {},
"outputs": [],
"source": [
@@ -1739,16 +1739,16 @@
},
{
"cell_type": "code",
- "execution_count": 72,
+ "execution_count": 77,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f9176ad9b38>"
+ "<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x1203a60a0>"
]
},
- "execution_count": 72,
+ "execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
@@ -1759,7 +1759,7 @@
},
{
"cell_type": "code",
- "execution_count": 73,
+ "execution_count": 78,
"metadata": {},
"outputs": [
{
@@ -1778,7 +1778,7 @@
},
{
"cell_type": "code",
- "execution_count": 74,
+ "execution_count": 79,
"metadata": {},
"outputs": [
{
@@ -1805,7 +1805,7 @@
},
{
"cell_type": "code",
- "execution_count": 75,
+ "execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
@@ -1814,7 +1814,7 @@
},
{
"cell_type": "code",
- "execution_count": 76,
+ "execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
@@ -1823,7 +1823,7 @@
},
{
"cell_type": "code",
- "execution_count": 77,
+ "execution_count": 82,
"metadata": {},
"outputs": [
{
@@ -1857,7 +1857,7 @@
},
{
"cell_type": "code",
- "execution_count": 78,
+ "execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
@@ -1871,7 +1871,7 @@
},
{
"cell_type": "code",
- "execution_count": 79,
+ "execution_count": 84,
"metadata": {},
"outputs": [],
"source": [
@@ -1881,7 +1881,7 @@
},
{
"cell_type": "code",
- "execution_count": 80,
+ "execution_count": 85,
"metadata": {},
"outputs": [
{
@@ -1901,7 +1901,7 @@
},
{
"cell_type": "code",
- "execution_count": 81,
+ "execution_count": 86,
"metadata": {},
"outputs": [
{
@@ -1911,11 +1911,11 @@
"+--------------------+--------------------+\n",
"| leftgeometry| rightgeometry|\n",
"+--------------------+--------------------+\n",
- "|POLYGON ((-87.285...|POINT (-87.28468 ...|\n",
- "|POLYGON ((-87.285...|POINT (-87.278485...|\n",
- "|POLYGON ((-87.285...|POINT (-87.280556...|\n",
- "|POLYGON ((-87.285...|POINT (-87.270187...|\n",
- "|POLYGON ((-87.285...|POINT (-87.268766...|\n",
+ "|POLYGON ((-87.229...|POINT (-87.105455...|\n",
+ "|POLYGON ((-87.229...|POINT (-87.10534 ...|\n",
+ "|POLYGON ((-87.229...|POINT (-87.160372...|\n",
+ "|POLYGON ((-87.229...|POINT (-87.204033...|\n",
+ "|POLYGON ((-87.229...|POINT (-87.204299...|\n",
"+--------------------+--------------------+\n",
"only showing top 5 rows\n",
"\n"
@@ -1928,16 +1928,16 @@
},
{
"cell_type": "code",
- "execution_count": 82,
+ "execution_count": 87,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "Row(leftgeometry=<shapely.geometry.polygon.Polygon object at 0x7f9176a4e470>, rightgeometry=<shapely.geometry.point.Point object at 0x7f9176a4e4a8>)"
+ "Row(leftgeometry=<shapely.geometry.polygon.Polygon object at 0x11ffdb100>, rightgeometry=<shapely.geometry.point.Point object at 0x11ffdb1c0>)"
]
},
- "execution_count": 82,
+ "execution_count": 87,
"metadata": {},
"output_type": "execute_result"
}
@@ -1955,7 +1955,7 @@
},
{
"cell_type": "code",
- "execution_count": 83,
+ "execution_count": 88,
"metadata": {},
"outputs": [],
"source": [
@@ -1964,7 +1964,7 @@
},
{
"cell_type": "code",
- "execution_count": 84,
+ "execution_count": 89,
"metadata": {},
"outputs": [
{
@@ -1974,11 +1974,11 @@
"+--------------------+--------------+--------------------+---------------+\n",
"| leftgeometry|left_user_data| rightgeometry|right_user_data|\n",
"+--------------------+--------------+--------------------+---------------+\n",
- "|POLYGON ((-87.285...| |POINT (-87.28468 ...| null|\n",
- "|POLYGON ((-87.285...| |POINT (-87.278485...| null|\n",
- "|POLYGON ((-87.285...| |POINT (-87.280556...| null|\n",
- "|POLYGON ((-87.285...| |POINT (-87.270187...| null|\n",
- "|POLYGON ((-87.285...| |POINT (-87.268766...| null|\n",
+ "|POLYGON ((-87.229...| |POINT (-87.105455...| null|\n",
+ "|POLYGON ((-87.229...| |POINT (-87.10534 ...| null|\n",
+ "|POLYGON ((-87.229...| |POINT (-87.160372...| null|\n",
+ "|POLYGON ((-87.229...| |POINT (-87.204033...| null|\n",
+ "|POLYGON ((-87.229...| |POINT (-87.204299...| null|\n",
"+--------------------+--------------+--------------------+---------------+\n",
"only showing top 5 rows\n",
"\n"
@@ -1998,7 +1998,7 @@
},
{
"cell_type": "code",
- "execution_count": 85,
+ "execution_count": 90,
"metadata": {},
"outputs": [],
"source": [
@@ -2009,7 +2009,7 @@
},
{
"cell_type": "code",
- "execution_count": 86,
+ "execution_count": 91,
"metadata": {},
"outputs": [],
"source": [
@@ -2019,7 +2019,7 @@
},
{
"cell_type": "code",
- "execution_count": 87,
+ "execution_count": 92,
"metadata": {},
"outputs": [
{
@@ -2046,7 +2046,7 @@
},
{
"cell_type": "code",
- "execution_count": 88,
+ "execution_count": 93,
"metadata": {},
"outputs": [
{
@@ -2065,7 +2065,7 @@
},
{
"cell_type": "code",
- "execution_count": 89,
+ "execution_count": 94,
"metadata": {},
"outputs": [],
"source": [
@@ -2076,7 +2076,7 @@
},
{
"cell_type": "code",
- "execution_count": 90,
+ "execution_count": 95,
"metadata": {},
"outputs": [
{
@@ -2103,7 +2103,7 @@
},
{
"cell_type": "code",
- "execution_count": 91,
+ "execution_count": 96,
"metadata": {},
"outputs": [
{
@@ -2138,7 +2138,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.1"
+ "version": "3.9.7"
}
},
"nbformat": 4,
diff --git a/binder/ApacheSedonaRaster_1.ipynb b/binder/ApacheSedonaRaster.ipynb
similarity index 75%
rename from binder/ApacheSedonaRaster_1.ipynb
rename to binder/ApacheSedonaRaster.ipynb
index 05bd5c1..a75b19e 100644
--- a/binder/ApacheSedonaRaster_1.ipynb
+++ b/binder/ApacheSedonaRaster.ipynb
@@ -2,17 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "import findspark\n",
- "findspark.init()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
+ "execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
@@ -40,9 +30,105 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 34,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:55:41 WARN UDTRegistration: Cannot register UDT for org.locationtech.jts.geom.Geometry, which is already registered.\n",
+ "21/10/08 19:55:41 WARN UDTRegistration: Cannot register UDT for org.locationtech.jts.index.SpatialIndex, which is already registered.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_pointfromtext replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_polygonfromtext replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_linestringfromtext replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geomfromtext replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geomfromwkt replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geomfromwkb replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geomfromgeojson replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_point replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_polygonfromenvelope replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_contains replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_intersects replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_within replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_distance replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_convexhull replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_npoints replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_buffer replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_envelope replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_length replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_area replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_centroid replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_transform replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_intersection replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_isvalid replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_precisionreduce replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_equals replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_touches replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_overlaps replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_crosses replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_issimple replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_makevalid replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_simplifypreservetopology replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_astext replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_asgeojson replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geometrytype replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_numgeometries replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_linemerge replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_azimuth replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_x replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_y replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_startpoint replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_boundary replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_minimumboundingradius replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_minimumboundingcircle replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_endpoint replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_exteriorring replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_geometryn replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_interiorringn replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_dump replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_dumppoints replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_isclosed replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_numinteriorrings replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_addpoint replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_removepoint replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_isring replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_flipcoordinates replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_linesubstring replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_lineinterpolatepoint replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_subdivideexplode replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_subdivide replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_normalizeddifference replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_mean replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_mode replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_fetchregion replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_greaterthan replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_greaterthanequal replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_lessthan replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_lessthanequal replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_addbands replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_subtractbands replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_dividebands replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_multiplyfactor replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_multiplybands replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_bitwiseand replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_bitwiseor replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_count replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_modulo replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_getband replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_squareroot replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_logicaldifference replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_logicalover replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_base64 replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_html replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_array replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function rs_normalize replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_union_aggr replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_envelope_aggr replaced a previously registered function.\n",
+ "21/10/08 19:55:41 WARN SimpleFunctionRegistry: The function st_intersection_aggr replaced a previously registered function.\n"
+ ]
+ }
+ ],
"source": [
"spark = SparkSession.\\\n",
" builder.\\\n",
@@ -50,10 +136,7 @@
" appName(\"Demo-app\").\\\n",
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName) .\\\n",
- " config(\"spark.executor.cores\", 3) .\\\n",
- " config(\"spark.driver.memory\", \"4G\") .\\\n",
- " config(\"spark.kryoserializer.buffer.max.value\", \"4096\") .\\\n",
- " config(\"spark.sql.crossJoin.enabled\", \"true\") .\\\n",
+ " config(\"spark.jars.packages\", \"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2-RC1\") .\\\n",
" getOrCreate()\n",
"\n",
"SedonaRegistrator.registerAll(spark)\n",
@@ -72,7 +155,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
@@ -82,7 +165,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 36,
"metadata": {
"scrolled": true
},
@@ -111,7 +194,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 37,
"metadata": {},
"outputs": [
{
@@ -121,8 +204,8 @@
"+--------------------+--------------------+------+-----+--------------------+-----+\n",
"| origin| Geom|height|width| data|bands|\n",
"+--------------------+--------------------+------+-----+--------------------+-----+\n",
- "|file:///home/hp/D...|POLYGON ((-58.702...| 32| 32|[1081.0, 1068.0, ...| 4|\n",
- "|file:///home/hp/D...|POLYGON ((-58.286...| 32| 32|[1151.0, 1141.0, ...| 4|\n",
+ "|file:///Users/jia...|POLYGON ((-58.702...| 32| 32|[1081.0, 1068.0, ...| 4|\n",
+ "|file:///Users/jia...|POLYGON ((-58.286...| 32| 32|[1151.0, 1141.0, ...| 4|\n",
"+--------------------+--------------------+------+-----+--------------------+-----+\n",
"\n"
]
@@ -142,7 +225,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 38,
"metadata": {},
"outputs": [
{
@@ -176,7 +259,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 39,
"metadata": {},
"outputs": [
{
@@ -202,7 +285,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 40,
"metadata": {},
"outputs": [
{
@@ -227,7 +310,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 41,
"metadata": {},
"outputs": [
{
@@ -252,7 +335,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 42,
"metadata": {},
"outputs": [
{
@@ -277,7 +360,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 43,
"metadata": {},
"outputs": [
{
@@ -303,7 +386,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 44,
"metadata": {},
"outputs": [
{
@@ -328,7 +411,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 45,
"metadata": {},
"outputs": [
{
@@ -353,7 +436,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 46,
"metadata": {},
"outputs": [
{
@@ -378,7 +461,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 47,
"metadata": {},
"outputs": [
{
@@ -403,7 +486,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 48,
"metadata": {},
"outputs": [
{
@@ -428,7 +511,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 49,
"metadata": {},
"outputs": [
{
@@ -453,7 +536,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 50,
"metadata": {},
"outputs": [
{
@@ -478,7 +561,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": 51,
"metadata": {},
"outputs": [
{
@@ -503,7 +586,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 52,
"metadata": {},
"outputs": [
{
@@ -528,7 +611,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 53,
"metadata": {},
"outputs": [
{
@@ -553,7 +636,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 54,
"metadata": {},
"outputs": [
{
@@ -578,7 +661,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 55,
"metadata": {},
"outputs": [
{
@@ -603,7 +686,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 56,
"metadata": {},
"outputs": [
{
@@ -628,7 +711,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 57,
"metadata": {},
"outputs": [
{
@@ -665,7 +748,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 58,
"metadata": {},
"outputs": [
{
@@ -689,14 +772,14 @@
"df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\")\n",
"\n",
"df = df.selectExpr(\"RS_GetBand(data,1,bands) as targetband\", \"height\", \"width\", \"bands\", \"Geom\")\n",
- "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Array(height*width,0), RS_Array(height*width, 0)) as red\",\"RS_Base64(height,width,RS_Array(height*width, 0), RS_Normalize(targetBand), RS_Array(height*width, 0)) as green\", \"RS_Base64(height,width,RS_Array(height*width, 0), RS_Array(height*width, 0), RS_Normalize(targetBand)) as blue\",\"RS_Base64(height,width,RS_Normalize(targetBand), RS_Normalize(targetBand),RS_Normalize(targetBand)) as [...]
+ "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Array(height*width,0.0), RS_Array(height*width, 0.0)) as red\",\"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Normalize(targetBand), RS_Array(height*width, 0.0)) as green\", \"RS_Base64(height,width,RS_Array(height*width, 0.0), RS_Array(height*width, 0.0), RS_Normalize(targetBand)) as blue\",\"RS_Base64(height,width,RS_Normalize(targetBand), RS_Normalize(targetBand),RS_Normalize(targ [...]
"df_HTML = df_base64.selectExpr(\"Geom\",\"RS_HTML(red) as RedBand\",\"RS_HTML(blue) as BlueBand\",\"RS_HTML(green) as GreenBand\", \"RS_HTML(RGB) as CombinedBand\")\n",
"df_HTML.show(5)"
]
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 59,
"metadata": {},
"outputs": [
{
@@ -717,18 +800,18 @@
" <tr>\n",
" <th>0</th>\n",
" <td>POLYGON ((-58.70271939504448 -34.41877544555479, -58.70277605822864 -34.42156988068061, -58.6994039180242 -34.42161679331493, -58.69934736692278 -34.4188223533111, -58.70271939504448 -34.41877544555479))</td>\n",
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFGElEQVR42l2X2W4jRRSG+zVhEu9b2+6O492xE8eJE8eZZLIxZNaISZgJIAHSSGwSFyziDoRAvABPgMQVQkhDneg70q+5+F3d5aqz/Gep6qgVRW/WAhoBpYA6Y5XnWkCRNb2AecCDgIuA44A44LOAPwM+DeiwZxU5hnxAhTGPbJsvB0Td8DNCuG3eFMWTAPs/hwH7rH0d8DTgVcAQPAm4ClgieAXDDRmQR1aeeTM+Mk8GojTBkHWQsNCepwGLgK2AMc87KP084N+A31ifBRnYyPGeYy7rDMyg3wTdBuyhbChGbDCOWechM3YOAt4L+CHgb8Y6jC6g2z1eQbmPZkw0QKCN3yDQvD5BiC38ACVbGLZLOM4ZzYlDDDWvUgy9YH0snq9iQAl5k [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFHklEQVR42l2X2W4jRRSG/ZowifetbbfttPexY8exsziTTBYzBGaJSMJMAIlBGolN4oJF3IEQiBfgCZC4QgjJVDXf8fyai9/VXa46y3+Wqk4kEs1VItFwqDrkHSqMJZ7LDjnWdBzmDg8dlg4nDoHDS4c/HD51aLFnEzkeGYciYwbZfr7gkGi7nwHC/eZtUTxa/f9/GgP2WfvK4YnDC4c+eOxw5bBA8AaGeyRBBlkZ5r3xsSc9URpiyBYIWeifJw4HDmOHIc+7KP3M4R+HX1mfAknYSPOeZi5lDEyh3wu6c9hDWV+MuM84ZJ2FzLNz6PCOw/cOfzFWYPQAus3jDZTb6I2JvW/AwtcI9F6fIsQv/AAlYwybEY4LRu/EEYZ6r+oYumR9IJ5vY [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFGElEQVR42l2X2W4jRRSG/ZpA4i3euu1uO+09duw4ceI4k0w2hswaMQlDAIlBGolN4oJF3IEQiBfgCZC4QghpqGq+Y/2ai9/VXa46y3+Wqs5kWpnXmaZD3aHkEDJWea45bLCm6zB3uO9w6XDiEDi8dPjD4VOHNnvWkeNRcKgwFpDt58sOmY77GSLcb94WxWMH/38eAw5Y+8rhicOHDgPw2OHaYYngNQz3yIICsgrMe+NTT/qiNMKQTRCx0D9PHRYOE4cRz7so/czhH4dfWZ8DWdjI855nLmcMzKDfC7pz2EfZQIzYYhyxzkLm2Tl0eNfhe4e/GEMYXUC3ebyGchu9Man3TVj4GoHe61OE+IXvo2SCYXuE44LRO3GEod6rGEMvWR+I5+sYU [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAGOElEQVR42k3XWUtXURQF8Ps1s3kwLRvNBm3O5kEtK0srmq3AgqAJemigtyIKv0CfIPBJJDjx27BE4Xj/99xz915r7eGc2+3Zs6ft3r27DQwMtC1btrTt27fXta+vr3739/e3zZs315r9+/e306dPt7GxsTY5OdmuXLnStm3b1l6+fNn+/PnT5ubm2r59++qdtWvXlh1j48aNbevWrXU12Dbf29vbuqGhoTY8PFzGvXz06NFlx0eOHGmeb9iwoQCcPXu21r5+/brNzMy0p0+ftkOHDtWYnp5u9+7daxcuXCjDa9asKeDGunXranDOlqt54DtMDh48uOx0586dBWTv3r013Fvo9/Hjx9u5c+fasWPH2uHDh+v3qVOnyumrV6/a0tJS+/XrV [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFGElEQVR42l2X2W4jRRSG+zVhEu9b2+6O492xE8eJE8eZZLIxZNaISZgJIAHSSGwSFyziDoRAvABPgMQVQkhDneg70q+5+F3d5aqz/Gep6qgVRW/WAhoBpYA6Y5XnWkCRNb2AecCDgIuA44A44LOAPwM+DeiwZxU5hnxAhTGPbJsvB0Td8DNCuG3eFMWTAPs/hwH7rH0d8DTgVcAQPAm4ClgieAXDDRmQR1aeeTM+Mk8GojTBkHWQsNCepwGLgK2AMc87KP084N+A31ifBRnYyPGeYy7rDMyg3wTdBuyhbChGbDCOWechM3YOAt4L+CHgb8Y6jC6g2z1eQbmPZkw0QKCN3yDQvD5BiC38ACVbGLZLOM4ZzYlDDDWvUgy9YH0snq9iQAl5k [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFHklEQVR42l2X2W4jRRSG/ZowifetbbfttPexY8exsziTTBYzBGaJSMJMAIlBGolN4oJF3IEQiBfgCZC4QgjJVDXf8fyai9/VXa46y3+Wqk4kEs1VItFwqDrkHSqMJZ7LDjnWdBzmDg8dlg4nDoHDS4c/HD51aLFnEzkeGYciYwbZfr7gkGi7nwHC/eZtUTxa/f9/GgP2WfvK4YnDC4c+eOxw5bBA8AaGeyRBBlkZ5r3xsSc9URpiyBYIWeifJw4HDmOHIc+7KP3M4R+HX1mfAknYSPOeZi5lDEyh3wu6c9hDWV+MuM84ZJ2FzLNz6PCOw/cOfzFWYPQAus3jDZTb6I2JvW/AwtcI9F6fIsQv/AAlYwybEY4LRu/EEYZ6r+oYumR9IJ5vY [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFGElEQVR42l2X2W4jRRSG/ZpA4i3euu1uO+09duw4ceI4k0w2hswaMQlDAIlBGolN4oJF3IEQiBfgCZC4QghpqGq+Y/2ai9/VXa46y3+Wqs5kWpnXmaZD3aHkEDJWea45bLCm6zB3uO9w6XDiEDi8dPjD4VOHNnvWkeNRcKgwFpDt58sOmY77GSLcb94WxWMH/38eAw5Y+8rhicOHDgPw2OHaYYngNQz3yIICsgrMe+NTT/qiNMKQTRCx0D9PHRYOE4cRz7so/czhH4dfWZ8DWdjI855nLmcMzKDfC7pz2EfZQIzYYhyxzkLm2Tl0eNfhe4e/GEMYXUC3ebyGchu9Man3TVj4GoHe61OE+IXvo2SCYXuE44LRO3GEod6rGEMvWR+I5+sYU [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAGOElEQVR42k3XWUtXURQF8Ps1s3kwLRvNBm3O5kEtK0srmq3AgqAJemigtyIKv0CfIPBJJDjx27BE4Xj/99xz915r7eGc2+3Zs6ft3r27DQwMtC1btrTt27fXta+vr3739/e3zZs315r9+/e306dPt7GxsTY5OdmuXLnStm3b1l6+fNn+/PnT5ubm2r59++qdtWvXlh1j48aNbevWrXU12Dbf29vbuqGhoTY8PFzGvXz06NFlx0eOHGmeb9iwoQCcPXu21r5+/brNzMy0p0+ftkOHDtWYnp5u9+7daxcuXCjDa9asKeDGunXranDOlqt54DtMDh48uOx0586dBWTv3r013Fvo9/Hjx9u5c+fasWPH2uHDh+v3qVOnyumrV6/a0tJS+/XrV [...]
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>POLYGON ((-58.28663657626114 -34.75858090620287, -58.28667994174946 -34.76137571668496, -58.28329340123002 -34.76141146033393, -58.28325014980316 -34.75861664615162, -58.28663657626114 -34.75858090620287))</td>\n",
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE+UlEQVR42l2XS4/bVBTH/R0p08nDieOx4zhO7LxfM5lH0xnoY9qhtKgMagEJVd0hIbaw4FuwR2zY8QXKvdHvoL+yOLF977nn/M/7JoiD4FPHUelo4Shx9NBR31HlqO3I8zQcHTtqOooctXgv2D9iP2Yt4sxDqIGsnaO/HL1ylDsKUgCkgMgR3GYtFiFeQR3lkfCMHPXg9We7Yog/EwIgRMczzpx4AP5n4miA1TnUEvKHP3P0AEsNkFc0dnTr6AVgauyF4oUO3976P7De8w49gDkAEhDFAEhBn/B+JACOAeQ9cO7oFMtCeI4B0YLfQNw7+hUA3uAbDUEf97UAMCGWU/bash/KM5LnhLMNqH4QzgGhWiDXey2owdjHpRFCvGc2vO842IGar [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE/ElEQVR42l2XyXLcVBSG+x0JjntQS62WWq0epJ4nu2M7HZuExCEJQwVTCVBFUeyootjCgrdgT7FhxwuYe5XvmL+8OC3p3jP8Z7y3a7VafFurdRwVjlaOEkcPHQ0clY7C2w88TUfHjlqOIkdt3ofsH7EfsxYh8xBqouvg6C9HbxzljmopAFJA5CgOWYtFiTfQwHgkPBNHfXi9bE8c8TIBAAJsPEem6wH4n5mjEV7nUFvIC3/k6AGeGiBvaOro2tFLwNTZCyQKHb6993/gvecdewBLACQgigGQgj7h/UgAHAPIR+CRoxM8C+A5BkQbfgNx4+hXAHiHrzQFA8LXBsCMXM7ZC2U/kGckzxmyTahxL50jUrVCr49aFbIGRnoo8kp8ZHa8HxDsQ [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE+ElEQVR42l2XyW7bVhSG9Y5NHWugSFGkKGogNU+2YjuK3aSJ3SQdkLpomgBFkF2Bott20bfovuimu75Aei/xHeOHF0ck7z3Df8Z7VavFtU+1jqPC0cpR4uiho4Gj0lHoyPM0HR07ajmKHLV5H7J/xH7MWoTMQ6iJroOjvx29dpQ7qqUASAGRozhkLRYl3kAD45HwTBz14fWyPXHEywQACLDxHJmuB+B/Zo5GeJ1DbSEv/JmjB3hqgLyhqaNrRzeAqbMXSBQ6fHvv/8R7zzv2AJYASEAUAyAFfcL7kQA4BpCPwCNHJ3gWwHMMiDb8BuLW0W8A8A5faQoGhK8NgBm5nLMXyn4gz0ieM2SbUONeOkekaoVeH7UqZA2M9FDklfjI7Hg/INiBW [...]
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAF7klEQVR42k3XyU5WSxQF4POOKCooSmODqKggNoCCiA2tghqUAJIY44zEMNWBb8HcOGHmC9TNt5PFhaSoU1W7Wbut+ruhoaE2ODjY7ty506amptrIyEg7d+5cu3nzZhsfH2+XL19uaPr7+9v58+fbxYsX25UrV9rAwEB9j42N1Xlvb2+d+7aHBg9Zhm+yFhYW2p8/f9rGxkYbHR1t3dWrVwuAGQibBCO2R2CEUNDX11fKKQjN3bt3240bN4oW77Vr104MwXPp0qUCYKZjeXm5eIaHh1vn3/3799utW7fKagAMSjIw9/T0tDNnzpSlAUTRvXv32urqaltbWyswFy5cqDPK4gUGWrP+9+/fZT3a27dvt+7BgwcFAGJgWAEAAujt+6Y4AOwDx [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE+UlEQVR42l2XS4/bVBTH/R0p08nDieOx4zhO7LxfM5lH0xnoY9qhtKgMagEJVd0hIbaw4FuwR2zY8QXKvdHvoL+yOLF977nn/M/7JoiD4FPHUelo4Shx9NBR31HlqO3I8zQcHTtqOooctXgv2D9iP2Yt4sxDqIGsnaO/HL1ylDsKUgCkgMgR3GYtFiFeQR3lkfCMHPXg9We7Yog/EwIgRMczzpx4AP5n4miA1TnUEvKHP3P0AEsNkFc0dnTr6AVgauyF4oUO3976P7De8w49gDkAEhDFAEhBn/B+JACOAeQ9cO7oFMtCeI4B0YLfQNw7+hUA3uAbDUEf97UAMCGWU/bash/KM5LnhLMNqH4QzgGhWiDXey2owdjHpRFCvGc2vO842IGar [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE/ElEQVR42l2XyXLcVBSG+x0JjntQS62WWq0epJ4nu2M7HZuExCEJQwVTCVBFUeyootjCgrdgT7FhxwuYe5XvmL+8OC3p3jP8Z7y3a7VafFurdRwVjlaOEkcPHQ0clY7C2w88TUfHjlqOIkdt3ofsH7EfsxYh8xBqouvg6C9HbxzljmopAFJA5CgOWYtFiTfQwHgkPBNHfXi9bE8c8TIBAAJsPEem6wH4n5mjEV7nUFvIC3/k6AGeGiBvaOro2tFLwNTZCyQKHb6993/gvecdewBLACQgigGQgj7h/UgAHAPIR+CRoxM8C+A5BkQbfgNx4+hXAHiHrzQFA8LXBsCMXM7ZC2U/kGckzxmyTahxL50jUrVCr49aFbIGRnoo8kp8ZHa8HxDsQ [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAE+ElEQVR42l2XyW7bVhSG9Y5NHWugSFGkKGogNU+2YjuK3aSJ3SQdkLpomgBFkF2Bott20bfovuimu75Aei/xHeOHF0ck7z3Df8Z7VavFtU+1jqPC0cpR4uiho4Gj0lHoyPM0HR07ajmKHLV5H7J/xH7MWoTMQ6iJroOjvx29dpQ7qqUASAGRozhkLRYl3kAD45HwTBz14fWyPXHEywQACLDxHJmuB+B/Zo5GeJ1DbSEv/JmjB3hqgLyhqaNrRzeAqbMXSBQ6fHvv/8R7zzv2AJYASEAUAyAFfcL7kQA4BpCPwCNHJ3gWwHMMiDb8BuLW0W8A8A5faQoGhK8NgBm5nLMXyn4gz0ieM2SbUONeOkekaoVeH7UqZA2M9FDklfjI7Hg/INiBW [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAF7klEQVR42k3XyU5WSxQF4POOKCooSmODqKggNoCCiA2tghqUAJIY44zEMNWBb8HcOGHmC9TNt5PFhaSoU1W7Wbut+ruhoaE2ODjY7ty506amptrIyEg7d+5cu3nzZhsfH2+XL19uaPr7+9v58+fbxYsX25UrV9rAwEB9j42N1Xlvb2+d+7aHBg9Zhm+yFhYW2p8/f9rGxkYbHR1t3dWrVwuAGQibBCO2R2CEUNDX11fKKQjN3bt3240bN4oW77Vr104MwXPp0qUCYKZjeXm5eIaHh1vn3/3799utW7fKagAMSjIw9/T0tDNnzpSlAUTRvXv32urqaltbWyswFy5cqDPK4gUGWrP+9+/fZT3a27dvt+7BgwcFAGJgWAEAAujt+6Y4AOwDx [...]
" </tr>\n",
" </tbody>\n",
"</table>"
@@ -754,7 +837,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 60,
"metadata": {},
"outputs": [
{
@@ -769,6 +852,13 @@
"+------+\n",
"\n"
]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:55:44 WARN SimpleFunctionRegistry: The function rs_sum replaced a previously registered function.\n"
+ ]
}
],
"source": [
@@ -790,10 +880,17 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 61,
"metadata": {},
"outputs": [
{
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:55:44 WARN SimpleFunctionRegistry: The function rs_maskvalues replaced a previously registered function.\n"
+ ]
+ },
+ {
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
@@ -808,12 +905,12 @@
" <tr>\n",
" <th>0</th>\n",
" <td>POLYGON ((-58.70271939504448 -34.41877544555479, -58.70277605822864 -34.42156988068061, -58.6994039180242 -34.42161679331493, -58.69934736692278 -34.4188223533111, -58.70271939504448 -34.41877544555479))</td>\n",
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFlklEQVR42l2Xa28bRRSG529CXV/XG6/t9S1rx/c4iZPGaS6N3RBK2oiGSwCJVqpEAYkPBcQ3EALxB/gFSHxCCGk5M/OMM+qH1zO73p1zznvec2ZWtZRSTUFNEAiqjBvMK4ISz3QFc6XSU8FKcCKIBC8Efwq+FGwK5J30viAABUEo7xbABjbKApUIBiwuL6uJZ3jM/3kcOLDPpq8EV4JPBX3wgeBasBDIwmlGUAJZC2M8z6iDirQDEona8ozGONIGMQ/q+bbggSw2FYwEer6L0ZeCfwW/wUoOZGFDG87hQJa5YWAH+mUhdSvYx1jfc2LIOLLPpcJGKu+kwk56KHgs+EHwN6MEk3ZxcIMU6IgzGHejdsZE32T8VnBI1GekRT/4oUCMqKl1L [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFlklEQVR42l2Xa28bRRSG529CXV/XG6/t9S1rx/c4iZPGaS6N3RBK2oiGSwCJVqpEAYkPBcQ3EALxB/gFSHxCCGk5M/OMM+qH1zO73p1zznvec2ZWtZRSTUFNEAiqjBvMK4ISz3QFc6XSU8FKcCKIBC8Efwq+FGwK5J30viAABUEo7xbABjbKApUIBiwuL6uJZ3jM/3kcOLDPpq8EV4JPBX3wgeBasBDIwmlGUAJZC2M8z6iDirQDEona8ozGONIGMQ/q+bbggSw2FYwEer6L0ZeCfwW/wUoOZGFDG87hQJa5YWAH+mUhdSvYx1jfc2LIOLLPpcJGKu+kwk56KHgs+EHwN6MEk3ZxcIMU6IgzGHejdsZE32T8VnBI1GekRT/4oUCMqKl1L [...]
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>POLYGON ((-58.28663657626114 -34.75858090620287, -58.28667994174946 -34.76137571668496, -58.28329340123002 -34.76141146033393, -58.28325014980316 -34.75861664615162, -58.28663657626114 -34.75858090620287))</td>\n",
- " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFfklEQVR42k2XSXPbRhCF8R/tSOJOQiRBECTBTaRILdZiybJjKbacpGynvKQqlcotVSlfk0P+Re6pXHLLH0C6e74R5vAEgFj69evXPaMojqKoI5gIVoKuYFcwFOSClkCfqQn2BPUoKtqCpkDPM4HcL3YEe5xnPCPvFLugxrcuBX8L7gWpIOpBoAeJ1AWxh3sE3wVKoEpwDSDPFPJMMRUMCK6k+oIugZVUAwINYnwtkHei/Yg/c8GIrFPQDKAvPxI8FuwEWWmgmeBWcAeZCvcagQodd23Z/0n2mtxYCRxAoAuZGAI9Mu5yvhMQ0KweocCp4EgwIehj7u+i1E5JInov+AIBTfg6LMEQ6ZsQUFJSy2jBvVZ53wI1gyz9Ud4pUgIqqpSEUlnQA [...]
+ " <td><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFfklEQVR42k2XSXPbRhCF8R/tSOJOQiRBECTBTaRILdZiybJjKbacpGynvKQqlcotVSlfk0P+Re6pXHLLH0C6e74R5vAEgFj69evXPaMojqKoI5gIVoKuYFcwFOSClkCfqQn2BPUoKtqCpkDPM4HcL3YEe5xnPCPvFLugxrcuBX8L7gWpIOpBoAeJ1AWxh3sE3wVKoEpwDSDPFPJMMRUMCK6k+oIugZVUAwINYnwtkHei/Yg/c8GIrFPQDKAvPxI8FuwEWWmgmeBWcAeZCvcagQodd23Z/0n2mtxYCRxAoAuZGAI9Mu5yvhMQ0KweocCp4EgwIehj7u+i1E5JInov+AIBTfg6LMEQ6ZsQUFJSy2jBvVZ53wI1gyz9Ud4pUgIqqpSEUlnQA [...]
" </tr>\n",
" </tbody>\n",
"</table>"
@@ -841,7 +938,7 @@
"spark.udf.register(\"RS_MaskValues\", maskValues)\n",
"\n",
"\n",
- "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetband), RS_Array(height*width,0), RS_Array(height*width, 0), RS_MaskValues(targetband,width,height)) as region\" )\n",
+ "df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetband), RS_Array(height*width,0.0), RS_Array(height*width, 0.0), RS_MaskValues(targetband,width,height)) as region\" )\n",
"df_HTML = df_base64.selectExpr(\"Geom\",\"RS_HTML(region) as selectedregion\")\n",
"display(HTML(df_HTML.limit(2).toPandas().to_html(escape=False)))\n"
]
@@ -863,7 +960,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.5"
+ "version": "3.9.7"
}
},
"nbformat": 4,
diff --git a/binder/ApacheSedonaSQL.ipynb b/binder/ApacheSedonaSQL.ipynb
index f3ec6f1..a8125e3 100644
--- a/binder/ApacheSedonaSQL.ipynb
+++ b/binder/ApacheSedonaSQL.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@@ -17,7 +17,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@@ -27,24 +27,126 @@
" config(\"spark.serializer\", KryoSerializer.getName). \\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName). \\\n",
" config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.0.0-incubating,org.datasyslab:geotools-wrapper:geotools-24.0'). \\\n",
+ " 'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2-RC1'). \\\n",
" getOrCreate()"
]
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:57:20 WARN UDTRegistration: Cannot register UDT for org.locationtech.jts.geom.Geometry, which is already registered.\n",
+ "21/10/08 19:57:20 WARN UDTRegistration: Cannot register UDT for org.locationtech.jts.index.SpatialIndex, which is already registered.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_pointfromtext replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_polygonfromtext replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_linestringfromtext replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geomfromtext replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geomfromwkt replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geomfromwkb replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geomfromgeojson replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_point replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_polygonfromenvelope replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_contains replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_intersects replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_within replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_distance replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_convexhull replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_npoints replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_buffer replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_envelope replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_length replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_area replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_centroid replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_transform replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_intersection replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_isvalid replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_precisionreduce replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_equals replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_touches replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_overlaps replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_crosses replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_issimple replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_makevalid replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_simplifypreservetopology replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_astext replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_asgeojson replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geometrytype replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_numgeometries replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_linemerge replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_azimuth replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_x replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_y replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_startpoint replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_boundary replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_minimumboundingradius replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_minimumboundingcircle replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_endpoint replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_exteriorring replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_geometryn replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_interiorringn replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_dump replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_dumppoints replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_isclosed replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function"
+ ]
+ },
+ {
"data": {
"text/plain": [
"True"
]
},
- "execution_count": 5,
+ "execution_count": 23,
"metadata": {},
"output_type": "execute_result"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " st_numinteriorrings replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_addpoint replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_removepoint replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_isring replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_flipcoordinates replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_linesubstring replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_lineinterpolatepoint replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_subdivideexplode replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_subdivide replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_normalizeddifference replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_mean replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_mode replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_fetchregion replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_greaterthan replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_greaterthanequal replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_lessthan replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_lessthanequal replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_addbands replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_subtractbands replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_dividebands replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_multiplyfactor replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_multiplybands replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_bitwiseand replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_bitwiseor replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_count replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_modulo replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_getband replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_squareroot replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_logicaldifference replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_logicalover replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_base64 replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_html replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_array replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function rs_normalize replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_union_aggr replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_envelope_aggr replaced a previously registered function.\n",
+ "21/10/08 19:57:20 WARN SimpleFunctionRegistry: The function st_intersection_aggr replaced a previously registered function.\n"
+ ]
}
],
"source": [
@@ -67,7 +169,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
@@ -109,7 +211,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 25,
"metadata": {},
"outputs": [
{
@@ -150,7 +252,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
@@ -191,7 +293,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 27,
"metadata": {},
"outputs": [
{
@@ -239,7 +341,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 28,
"metadata": {},
"outputs": [
{
@@ -247,25 +349,32 @@
"output_type": "stream",
"text": [
"== Physical Plan ==\n",
- "DistanceJoin pointshape1#261: geometry, pointshape2#287: geometry, 2.0, false\n",
- ":- Project [st_point(cast(_c0#255 as decimal(24,20)), cast(_c1#256 as decimal(24,20))) AS pointshape1#261, abc AS name1#262]\n",
- ": +- FileScan csv [_c0#255,_c1#256] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[file:/Users/jiayu/GitHub/GeoSpark-datasys-repo/binder/data/testpoint.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string,_c1:string>\n",
- "+- Project [st_point(cast(_c0#281 as decimal(24,20)), cast(_c1#282 as decimal(24,20))) AS pointshape2#287, def AS name2#288]\n",
- " +- FileScan csv [_c0#281,_c1#282] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[file:/Users/jiayu/GitHub/GeoSpark-datasys-repo/binder/data/testpoint.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string,_c1:string>\n",
+ "DistanceJoin pointshape1#651: geometry, pointshape2#675: geometry, 2.0, false\n",
+ ":- Project [st_point(cast(_c0#647 as decimal(24,20)), cast(_c1#648 as decimal(24,20))) AS pointshape1#651, abc AS name1#652]\n",
+ ": +- FileScan csv [_c0#647,_c1#648] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[file:/Users/jiayu/GitHub/GeoSpark-datasys-repo/binder/data/testpoint.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string,_c1:string>\n",
+ "+- Project [st_point(cast(_c0#671 as decimal(24,20)), cast(_c1#672 as decimal(24,20))) AS pointshape2#675, def AS name2#676]\n",
+ " +- FileScan csv [_c0#671,_c1#672] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[file:/Users/jiayu/GitHub/GeoSpark-datasys-repo/binder/data/testpoint.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string,_c1:string>\n",
"\n",
"\n",
"+-----------------+-----+-----------------+-----+\n",
"| pointshape1|name1| pointshape2|name2|\n",
"+-----------------+-----+-----------------+-----+\n",
"|POINT (1.1 101.1)| abc|POINT (1.1 101.1)| def|\n",
- "|POINT (1.1 101.1)| abc|POINT (2.1 102.1)| def|\n",
"|POINT (2.1 102.1)| abc|POINT (1.1 101.1)| def|\n",
+ "|POINT (1.1 101.1)| abc|POINT (2.1 102.1)| def|\n",
"|POINT (2.1 102.1)| abc|POINT (2.1 102.1)| def|\n",
- "|POINT (2.1 102.1)| abc|POINT (3.1 103.1)| def|\n",
+ "|POINT (3.1 103.1)| abc|POINT (2.1 102.1)| def|\n",
"+-----------------+-----+-----------------+-----+\n",
"only showing top 5 rows\n",
"\n"
]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:57:21 WARN JoinQuery: UseIndex is true, but no index exists. Will build index on the fly.\n"
+ ]
}
],
"source": [
@@ -309,7 +418,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
@@ -322,7 +431,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 30,
"metadata": {},
"outputs": [
{
@@ -345,7 +454,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 31,
"metadata": {},
"outputs": [
{
@@ -372,7 +481,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
@@ -381,7 +490,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
@@ -398,7 +507,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 34,
"metadata": {},
"outputs": [
{
@@ -425,7 +534,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
@@ -434,7 +543,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
@@ -449,37 +558,44 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 37,
"metadata": {},
"outputs": [
{
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:57:23 WARN JoinQuery: UseIndex is true, but no index exists. Will build index on the fly.\n"
+ ]
+ },
+ {
"name": "stdout",
"output_type": "stream",
"text": [
- "+---------+----------+--------------------+\n",
- "| id_1| id_2| geom|\n",
- "+---------+----------+--------------------+\n",
- "|197624402| 197624402|POINT (-3383818.5...|\n",
- "|197663196| 197663196|POINT (-3383367.1...|\n",
- "|197953474| 197953474|POINT (-3383763.3...|\n",
- "|262310516| 262310516|POINT (-3384257.6...|\n",
- "|262310516|1074233123|POINT (-3384257.6...|\n",
- "|270281140| 270281140|POINT (-3385421.2...|\n",
- "|270281140|1074232906|POINT (-3385421.2...|\n",
- "|270306609| 270306609|POINT (-3383982.8...|\n",
- "|270306746| 270306746|POINT (-3383898.4...|\n",
- "|293896571| 293896571|POINT (-3385029.0...|\n",
- "|293896571|3256728465|POINT (-3385029.0...|\n",
- "|360178884| 360178884|POINT (-3377483.1...|\n",
- "|360178897| 360178897|POINT (-3374350.0...|\n",
- "|360178897| 360178897|POINT (-3374350.0...|\n",
- "|360178897|5546280698|POINT (-3374350.0...|\n",
- "|360178897|5546280699|POINT (-3374350.0...|\n",
- "|360178897| 360178897|POINT (-3374350.0...|\n",
- "|360178897| 360178897|POINT (-3374350.0...|\n",
- "|360178897|5546280698|POINT (-3374350.0...|\n",
- "|360178897|5546280699|POINT (-3374350.0...|\n",
- "+---------+----------+--------------------+\n",
+ "+----------+---------+--------------------+\n",
+ "| id_1| id_2| geom|\n",
+ "+----------+---------+--------------------+\n",
+ "| 197624402|197624402|POINT (-3383818.5...|\n",
+ "| 197663196|197663196|POINT (-3383367.1...|\n",
+ "| 197953474|197953474|POINT (-3383763.3...|\n",
+ "| 262310516|262310516|POINT (-3384257.6...|\n",
+ "|1074233123|262310516|POINT (-3384262.1...|\n",
+ "| 270281140|270281140|POINT (-3385421.2...|\n",
+ "|1074232906|270281140|POINT (-3385408.6...|\n",
+ "| 270306609|270306609|POINT (-3383982.8...|\n",
+ "| 270306746|270306746|POINT (-3383898.4...|\n",
+ "| 293896571|293896571|POINT (-3385029.0...|\n",
+ "|3256728465|293896571|POINT (-3385002.4...|\n",
+ "| 360178884|360178884|POINT (-3377483.1...|\n",
+ "| 360178897|360178897|POINT (-3374350.0...|\n",
+ "| 360178897|360178897|POINT (-3374350.0...|\n",
+ "|5546280698|360178897|POINT (-3374344.9...|\n",
+ "|5546280699|360178897|POINT (-3374339.7...|\n",
+ "| 360178897|360178897|POINT (-3374350.0...|\n",
+ "| 360178897|360178897|POINT (-3374350.0...|\n",
+ "|5546280698|360178897|POINT (-3374344.9...|\n",
+ "|5546280699|360178897|POINT (-3374339.7...|\n",
+ "+----------+---------+--------------------+\n",
"only showing top 20 rows\n",
"\n"
]
@@ -498,16 +614,25 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 38,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:57:23 WARN JoinQuery: UseIndex is true, but no index exists. Will build index on the fly.\n",
+ " \r"
+ ]
+ }
+ ],
"source": [
"df = neighbours_within_1000m.toPandas()"
]
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
@@ -516,7 +641,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 40,
"metadata": {},
"outputs": [
{
@@ -572,9 +697,9 @@
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
- " <td>262310516</td>\n",
" <td>1074233123</td>\n",
- " <td>POINT (-3384257.682 4178033.053)</td>\n",
+ " <td>262310516</td>\n",
+ " <td>POINT (-3384262.187 4178036.442)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
@@ -584,21 +709,21 @@
" </tr>\n",
" <tr>\n",
" <th>45314</th>\n",
- " <td>6620325385</td>\n",
- " <td>6620325385</td>\n",
- " <td>POINT (-3215183.489 4307887.823)</td>\n",
+ " <td>6617406900</td>\n",
+ " <td>6617406900</td>\n",
+ " <td>POINT (-3202030.997 4313940.216)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45315</th>\n",
- " <td>6631077531</td>\n",
- " <td>6631077531</td>\n",
- " <td>POINT (-3227737.389 4306566.622)</td>\n",
+ " <td>6617371185</td>\n",
+ " <td>6619204985</td>\n",
+ " <td>POINT (-3224898.369 4313308.131)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45316</th>\n",
- " <td>6736467188</td>\n",
- " <td>6736467188</td>\n",
- " <td>POINT (-3242695.331 4298828.321)</td>\n",
+ " <td>6619204985</td>\n",
+ " <td>6619204985</td>\n",
+ " <td>POINT (-3224887.255 4313298.404)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45317</th>\n",
@@ -623,18 +748,18 @@
"1 197663196 197663196 POINT (-3383367.151 4179427.096)\n",
"2 197953474 197953474 POINT (-3383763.332 4179408.785)\n",
"3 262310516 262310516 POINT (-3384257.682 4178033.053)\n",
- "4 262310516 1074233123 POINT (-3384257.682 4178033.053)\n",
+ "4 1074233123 262310516 POINT (-3384262.187 4178036.442)\n",
"... ... ... ...\n",
- "45314 6620325385 6620325385 POINT (-3215183.489 4307887.823)\n",
- "45315 6631077531 6631077531 POINT (-3227737.389 4306566.622)\n",
- "45316 6736467188 6736467188 POINT (-3242695.331 4298828.321)\n",
+ "45314 6617406900 6617406900 POINT (-3202030.997 4313940.216)\n",
+ "45315 6617371185 6619204985 POINT (-3224898.369 4313308.131)\n",
+ "45316 6619204985 6619204985 POINT (-3224887.255 4313298.404)\n",
"45317 6736772185 6736772185 POINT (-3204857.139 4313763.361)\n",
"45318 6817416704 6817416704 POINT (-3214549.268 4314872.904)\n",
"\n",
"[45319 rows x 3 columns]"
]
},
- "execution_count": 20,
+ "execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
@@ -642,13 +767,6 @@
"source": [
"gdf"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -667,7 +785,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.1"
+ "version": "3.9.7"
}
},
"nbformat": 4,
diff --git a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
index 2444575..3758ed8 100644
--- a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
+++ b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
@@ -32,7 +32,71 @@
"cell_type": "code",
"execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:58:06 WARN Utils: Your hostname, Jia-MacBook-Pro.local resolves to a loopback address: 127.0.0.1; using 192.168.0.34 instead (on interface en0)\n",
+ "21/10/08 19:58:06 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ ":: loading settings :: url = jar:file:/Users/jiayu/Downloads/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar!/org/apache/ivy/core/settings/ivysettings.xml\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Ivy Default Cache set to: /Users/jiayu/.ivy2/cache\n",
+ "The jars for the packages stored in: /Users/jiayu/.ivy2/jars\n",
+ "org.apache.sedona#sedona-python-adapter-3.0_2.12 added as a dependency\n",
+ "org.datasyslab#geotools-wrapper added as a dependency\n",
+ ":: resolving dependencies :: org.apache.spark#spark-submit-parent-15133089-5026-4f0c-8975-860f756f148d;1.0\n",
+ "\tconfs: [default]\n",
+ "\tfound org.apache.sedona#sedona-python-adapter-3.0_2.12;1.1.0-incubating in central\n",
+ "\tfound org.locationtech.jts#jts-core;1.18.0 in central\n",
+ "\tfound org.wololo#jts2geojson;0.16.1 in central\n",
+ "\tfound com.fasterxml.jackson.core#jackson-databind;2.12.2 in central\n",
+ "\tfound com.fasterxml.jackson.core#jackson-annotations;2.12.2 in central\n",
+ "\tfound com.fasterxml.jackson.core#jackson-core;2.12.2 in central\n",
+ "\tfound org.apache.sedona#sedona-core-3.0_2.12;1.1.0-incubating in central\n",
+ "\tfound org.apache.sedona#sedona-sql-3.0_2.12;1.1.0-incubating in central\n",
+ "\tfound org.datasyslab#geotools-wrapper;1.1.0-25.2-RC1 in central\n",
+ ":: resolution report :: resolve 315ms :: artifacts dl 10ms\n",
+ "\t:: modules in use:\n",
+ "\tcom.fasterxml.jackson.core#jackson-annotations;2.12.2 from central in [default]\n",
+ "\tcom.fasterxml.jackson.core#jackson-core;2.12.2 from central in [default]\n",
+ "\tcom.fasterxml.jackson.core#jackson-databind;2.12.2 from central in [default]\n",
+ "\torg.apache.sedona#sedona-core-3.0_2.12;1.1.0-incubating from central in [default]\n",
+ "\torg.apache.sedona#sedona-python-adapter-3.0_2.12;1.1.0-incubating from central in [default]\n",
+ "\torg.apache.sedona#sedona-sql-3.0_2.12;1.1.0-incubating from central in [default]\n",
+ "\torg.datasyslab#geotools-wrapper;1.1.0-25.2-RC1 from central in [default]\n",
+ "\torg.locationtech.jts#jts-core;1.18.0 from central in [default]\n",
+ "\torg.wololo#jts2geojson;0.16.1 from central in [default]\n",
+ "\t:: evicted modules:\n",
+ "\torg.locationtech.jts#jts-core;1.18.1 by [org.locationtech.jts#jts-core;1.18.0] in [default]\n",
+ "\t---------------------------------------------------------------------\n",
+ "\t| | modules || artifacts |\n",
+ "\t| conf | number| search|dwnlded|evicted|| number|dwnlded|\n",
+ "\t---------------------------------------------------------------------\n",
+ "\t| default | 10 | 0 | 0 | 1 || 9 | 0 |\n",
+ "\t---------------------------------------------------------------------\n",
+ ":: retrieving :: org.apache.spark#spark-submit-parent-15133089-5026-4f0c-8975-860f756f148d\n",
+ "\tconfs: [default]\n",
+ "\t0 artifacts copied, 9 already retrieved (0kB/13ms)\n",
+ "21/10/08 19:58:06 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
+ "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
+ "Setting default log level to \"WARN\".\n",
+ "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
+ " \r"
+ ]
+ }
+ ],
"source": [
"spark = SparkSession.builder.\\\n",
" master(\"local[*]\").\\\n",
@@ -40,8 +104,8 @@
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName) .\\\n",
" config('spark.jars.packages',\n",
- " 'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.0.0-incubating,'\n",
- " 'org.datasyslab:geotools-wrapper:geotools-24.0'). \\\n",
+ " 'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,'\n",
+ " 'org.datasyslab:geotools-wrapper:1.1.0-25.2-RC1'). \\\n",
" getOrCreate()\n",
"SedonaRegistrator.registerAll(spark)\n",
"sc = spark.sparkContext\n",
@@ -163,6 +227,13 @@
" |-- NAME_ZH: string (nullable = true)\n",
"\n"
]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:58:13 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
+ ]
}
],
"source": [
@@ -280,32 +351,39 @@
"metadata": {},
"outputs": [
{
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "21/10/08 19:58:15 WARN JoinQuery: UseIndex is true, but no index exists. Will build index on the fly.\n"
+ ]
+ },
+ {
"name": "stdout",
"output_type": "stream",
"text": [
"+--------------------+--------------------+--------------------+--------------------+\n",
"| country_geom| NAME_EN| airport_geom| name|\n",
"+--------------------+--------------------+--------------------+--------------------+\n",
- "|MULTIPOLYGON (((3...|Zimbabwe ...|POINT (31.1014 -1...|Harare Int'l ...|\n",
- "|MULTIPOLYGON (((3...|Zimbabwe ...|POINT (28.6225520...|Bulawayo ...|\n",
- "|MULTIPOLYGON (((3...|Zambia ...|POINT (28.4455443...|Lusaka Int'l ...|\n",
- "|MULTIPOLYGON (((5...|Yemen ...|POINT (45.030602 ...|Aden Int'l ...|\n",
- "|MULTIPOLYGON (((5...|Yemen ...|POINT (44.2246467...|Sanaa Int'l ...|\n",
- "|MULTIPOLYGON (((1...|Vietnam ...|POINT (105.803759...|Noi Bai ...|\n",
- "|MULTIPOLYGON (((1...|Vietnam ...|POINT (106.664246...|Tan Son Nhat ...|\n",
- "|MULTIPOLYGON (((-...|Venezuela ...|POINT (-67.005748...|Simon Bolivar Int...|\n",
- "|MULTIPOLYGON (((-...|Venezuela ...|POINT (-71.723768...|La Chinita Int'l ...|\n",
- "|MULTIPOLYGON (((-...|Venezuela ...|POINT (-67.922361...|Zim Valencia ...|\n",
- "|MULTIPOLYGON (((7...|Uzbekistan ...|POINT (69.2666137...|Tashkent Int'l ...|\n",
- "|POLYGON ((-53.370...|Uruguay ...|POINT (-56.026636...|Carrasco Int'l ...|\n",
- "|MULTIPOLYGON (((-...|Puerto Rico ...|POINT (-66.004229...|Luis Munoz Marin ...|\n",
+ "|MULTIPOLYGON (((1...|Taiwan ...|POINT (121.231370...|Taoyuan ...|\n",
+ "|MULTIPOLYGON (((5...|Netherlands ...|POINT (4.76437693...|Schiphol ...|\n",
+ "|POLYGON ((103.969...|Singapore ...|POINT (103.986413...|Singapore Changi ...|\n",
+ "|MULTIPOLYGON (((-...|United Kingdom ...|POINT (-0.4531566...|London Heathrow ...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-149.98172...|Anchorage Int'l ...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-84.425397...|Hartsfield-Jackso...|\n",
+ "|MULTIPOLYGON (((1...|People's Republic...|POINT (116.588174...|Beijing Capital ...|\n",
+ "|MULTIPOLYGON (((-...|Colombia ...|POINT (-74.143371...|Eldorado Int'l ...|\n",
+ "|MULTIPOLYGON (((6...|India ...|POINT (72.8745639...|Chhatrapati Shiva...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-71.016406...|Gen E L Logan Int...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-76.668642...|Baltimore-Washing...|\n",
+ "|POLYGON ((36.8713...|Egypt ...|POINT (31.3997430...|Cairo Int'l ...|\n",
+ "|POLYGON ((-2.2196...|Morocco ...|POINT (-7.6632188...|Casablanca-Anfa ...|\n",
+ "|MULTIPOLYGON (((-...|Venezuela ...|POINT (-67.005748...|Simon Bolivar Int...|\n",
+ "|MULTIPOLYGON (((2...|South Africa ...|POINT (18.5976565...|Cape Town Int'l ...|\n",
+ "|MULTIPOLYGON (((1...|People's Republic...|POINT (103.956136...|Chengdushuang Liu...|\n",
+ "|MULTIPOLYGON (((6...|India ...|POINT (77.0878362...|Indira Gandhi Int...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-104.67379...|Denver Int'l ...|\n",
"|MULTIPOLYGON (((-...|United States of ...|POINT (-97.040371...|Dallas-Ft. Worth ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-74.177147...|Newark Int'l ...|\n",
+ "|MULTIPOLYGON (((1...|Thailand ...|POINT (100.602578...|Don Muang Int'l ...|\n",
"+--------------------+--------------------+--------------------+--------------------+\n",
"only showing top 20 rows\n",
"\n",
@@ -360,6 +438,16 @@
},
"outputs": [
{
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\r",
+ "[Stage 19:=================================================> (6 + 1) / 7]\r",
+ "\r",
+ " \r"
+ ]
+ },
+ {
"name": "stdout",
"output_type": "stream",
"text": [
@@ -423,7 +511,7 @@
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAACXCAYAAAAGX2ymAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIEUlEQVR4nO29eZQk6VXY+7sRudWStVfve0/PPpqtNTPakITQCtosIQlkMX5PtgyGB5hn/CSwHxzAWOYY8+RnDkLCegyyrEESkhBC1ooWRiDNotn3npmeXqb37qquNZeI+/64X+RSlZmVtWRlZnX8zslTmZEZEV9GRX73u7uoKjExMTExMUvhtXsAMTExMTHdQSwwYmJiYmKaIhYYMTExMTFNEQuMmJiYmJimiAVGTExMTExTJNo9gErGxsZ0z5497R5GTExMzJpz3333nVXV8VrvXSais6s49gn4mqq+YRWHaIqO [...]
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAACXCAYAAAAGX2ymAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABIAElEQVR4nO29eZgsaVng+3sjIpdasvY6+3769N50033obnYQQUDZBgTUUeZeZhgdvep4x7mg4+ijjpfxGa8Pc8dHBYcrOgyIKIjIsMpio9ALvTe9nO4+fZY++zlVp9ZcIt77x/tFLlWZWVlrZtaJ3/PkU5mRGRFfRkV+7/fuoqokJCQkJCQshdfuASQkJCQkdAeJwEhISEhIaIlEYCQkJCQktEQiMBISEhISWiIRGAkJCQkJLRG0ewDVjI2N6b59+9o9jISEhIQ157777juvquP13rtKRGdXcexT8CVVff0qDtES [...]
"text/plain": [
"<Figure size 432x288 with 2 Axes>"
]
@@ -452,13 +540,6 @@
" ax=ax\n",
")"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -477,7 +558,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.1"
+ "version": "3.9.7"
}
},
"nbformat": 4,
diff --git a/binder/Pipfile b/binder/Pipfile
index f126822..47b80f4 100644
--- a/binder/Pipfile
+++ b/binder/Pipfile
@@ -13,10 +13,10 @@ pytest-cov = "*"
[packages]
pandas="*"
geopandas="==0.6.0"
-pyspark="==3.0.1"
+pyspark="==3.1.2"
attrs="*"
ipykernel = "*"
-apache-sedona="==1.0.1"
+apache-sedona="==1.1.0"
matplotlib = "*"
descartes = "*"
diff --git a/binder/postBuild b/binder/postBuild
index 1c72262..9594969 100644
--- a/binder/postBuild
+++ b/binder/postBuild
@@ -1,6 +1,6 @@
#Download Apache Spark
-wget https://archive.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
-tar -xzf spark-3.0.1-bin-hadoop2.7.tgz
+wget https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
+tar -xzf spark-3.1.2-bin-hadoop3.2.tgz
#Tidy up
-rm spark-3.0.1-bin-hadoop2.7.tgz
\ No newline at end of file
+rm spark-3.1.2-bin-hadoop3.2.tgz
\ No newline at end of file
diff --git a/binder/start b/binder/start
index bf43357..59e5ccf 100644
--- a/binder/start
+++ b/binder/start
@@ -1,6 +1,6 @@
#!/bin/bash
-SPARK_HOME=$HOME/spark-3.0.1-bin-hadoop2.7
+SPARK_HOME=$HOME/spark-3.1.2-bin-hadoop3.2
export PATH=$SPARK_HOME/bin:$PATH
export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
export PYSPARK_SUBMIT_ARGS="--master local[*] pyspark-shell"