You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2021/07/02 06:21:18 UTC
[incubator-sedona] branch master updated: [SEDONA-30] Changes to
Geotiff Loader (#531)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new c363cd8 [SEDONA-30] Changes to Geotiff Loader (#531)
c363cd8 is described below
commit c363cd894759b0ecb1e82dec653f23a682f692d6
Author: shantanuaggarwal2695 <sh...@gmail.com>
AuthorDate: Thu Jul 1 23:21:11 2021 -0700
[SEDONA-30] Changes to Geotiff Loader (#531)
Co-authored-by: Jia Yu <ji...@gmail.com>
Co-authored-by: Jia Yu <ji...@apache.org>
---
binder/ApacheSedonaRaster_1.ipynb | 18 +++++++++---------
docs/api/sql/Raster-loader.md | 2 +-
.../apache/spark/sql/sedona_sql/io/GeotiffSchema.scala | 4 ++--
.../scala/org/apache/sedona/sql/rasterIOTest.scala | 6 +++---
4 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/binder/ApacheSedonaRaster_1.ipynb b/binder/ApacheSedonaRaster_1.ipynb
index 1220a51..05bd5c1 100644
--- a/binder/ApacheSedonaRaster_1.ipynb
+++ b/binder/ApacheSedonaRaster_1.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -12,7 +12,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -40,7 +40,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -72,7 +72,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -82,7 +82,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 9,
"metadata": {
"scrolled": true
},
@@ -94,7 +94,7 @@
"root\n",
" |-- image: struct (nullable = true)\n",
" | |-- origin: string (nullable = true)\n",
- " | |-- Geometry: geometry (nullable = true)\n",
+ " | |-- wkt: string (nullable = true)\n",
" | |-- height: integer (nullable = true)\n",
" | |-- width: integer (nullable = true)\n",
" | |-- nBands: integer (nullable = true)\n",
@@ -111,7 +111,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -129,7 +129,7 @@
}
],
"source": [
- "df = df.selectExpr(\"image.origin as origin\",\"image.Geometry as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\")\n",
+ "df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\")\n",
"df.show(5)"
]
},
@@ -686,7 +686,7 @@
"''' Plotting images as a dataframe using geotiff Dataframe.'''\n",
"\n",
"df = spark.read.format(\"geotiff\").option(\"dropInvalid\",True).load(DATA_DIR)\n",
- "df = df.selectExpr(\"image.origin as origin\",\"image.Geometry as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\")\n",
+ "df = df.selectExpr(\"image.origin as origin\",\"ST_GeomFromWkt(image.wkt) as Geom\", \"image.height as height\", \"image.width as width\", \"image.data as data\", \"image.nBands as bands\")\n",
"\n",
"df = df.selectExpr(\"RS_GetBand(data,1,bands) as targetband\", \"height\", \"width\", \"bands\", \"Geom\")\n",
"df_base64 = df.selectExpr(\"Geom\", \"RS_Base64(height,width,RS_Normalize(targetBand), RS_Array(height*width,0), RS_Array(height*width, 0)) as red\",\"RS_Base64(height,width,RS_Array(height*width, 0), RS_Normalize(targetBand), RS_Array(height*width, 0)) as green\", \"RS_Base64(height,width,RS_Array(height*width, 0), RS_Array(height*width, 0), RS_Normalize(targetBand)) as blue\",\"RS_Base64(height,width,RS_Normalize(targetBand), RS_Normalize(targetBand),RS_Normalize(targetBand)) as [...]
diff --git a/docs/api/sql/Raster-loader.md b/docs/api/sql/Raster-loader.md
index f4197b1..9d0b9b7 100644
--- a/docs/api/sql/Raster-loader.md
+++ b/docs/api/sql/Raster-loader.md
@@ -32,7 +32,7 @@ Output:
You can also select sub-attributes individually to construct a new DataFrame
```Scala
-geotiffDF = geotiffDF.selectExpr("image.origin as origin","image.Geometry as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
+geotiffDF = geotiffDF.selectExpr("image.origin as origin","ST_GeomFromWkt(image.wkt) as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
geotiffDF.createOrReplaceTempView("GeotiffDataframe")
geotiffDF.show()
```
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/io/GeotiffSchema.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/io/GeotiffSchema.scala
index bafa0f2..91b1c10 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/io/GeotiffSchema.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/io/GeotiffSchema.scala
@@ -40,7 +40,7 @@ object GeotiffSchema {
*/
val columnSchema = StructType(
StructField("origin", StringType, true) ::
- StructField("Geometry", GeometryUDT, true) ::
+ StructField("wkt", StringType, true) ::
StructField("height", IntegerType, false) ::
StructField("width", IntegerType, false) ::
StructField("nBands", IntegerType, false) ::
@@ -185,7 +185,7 @@ object GeotiffSchema {
}
}
// the internal "Row" is needed, because the image is a single DataFrame column
- Some(Row(Row(origin, polygon, height, width, nBands, decoded)))
+ Some(Row(Row(origin, polygon.toText, height, width, nBands, decoded)))
}
}
diff --git a/sql/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala b/sql/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
index c23a1f7..8f0f809 100644
--- a/sql/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
+++ b/sql/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
@@ -31,7 +31,7 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen
describe("Raster IO test") {
it("Should Pass geotiff loading") {
var df = sparkSession.read.format("geotiff").option("dropInvalid", true).load(rasterdatalocation)
- df = df.selectExpr("image.origin as origin","image.Geometry as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
+ df = df.selectExpr("image.origin as origin","ST_GeomFromWkt(image.wkt) as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
assert(df.first().getAs[Geometry](1).toText == "POLYGON ((-117.64141128097314 33.94356351407699, -117.64141128097314 33.664978146501284, -117.30939395196258 33.664978146501284," +
" -117.30939395196258 33.94356351407699, -117.64141128097314 33.94356351407699))")
assert(df.first().getInt(2) == 517)
@@ -53,7 +53,7 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen
it("should pass RS_Base64") {
var df = sparkSession.read.format("geotiff").option("dropInvalid", true).load(resourceFolder + "raster/")
- df = df.selectExpr("image.origin as origin","image.Geometry as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
+ df = df.selectExpr("image.origin as origin","ST_GeomFromWkt(image.wkt) as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
df = df.selectExpr("RS_GetBand(data, 1, bands) as targetBand", "width","height")
df.createOrReplaceTempView("geotiff")
df = sparkSession.sql("Select RS_base64(height, width, targetBand, RS_Array(height*width, 0), RS_Array(height*width, 0)) as encodedstring from geotiff")
@@ -62,7 +62,7 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen
it("should pass RS_HTML") {
var df = sparkSession.read.format("geotiff").option("dropInvalid", true).load(resourceFolder + "raster/")
- df = df.selectExpr("image.origin as origin","image.Geometry as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
+ df = df.selectExpr("image.origin as origin","ST_GeomFromWkt(image.wkt) as Geom", "image.height as height", "image.width as width", "image.data as data", "image.nBands as bands")
df = df.selectExpr("RS_GetBand(data, 1, bands) as targetBand", "width","height")
df.createOrReplaceTempView("geotiff")
df = sparkSession.sql("Select RS_base64(height, width, targetBand, RS_Array(height*width, 0.0), RS_Array(height*width, 0.0)) as encodedstring from geotiff")