You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/08/08 04:35:42 UTC
[sedona] branch fix-bug-docker-latest created (now a232978e)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch fix-bug-docker-latest
in repository https://gitbox.apache.org/repos/asf/sedona.git
at a232978e Fix a bug for building the docker for the latest image
This branch includes the following new commits:
new a232978e Fix a bug for building the docker for the latest image
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[sedona] 01/01: Fix a bug for building the docker for the latest image
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch fix-bug-docker-latest
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit a232978ec0328e7c4c5a2ccf71850660b446af8c
Author: Jia Yu <ji...@apache.org>
AuthorDate: Mon Aug 7 21:35:30 2023 -0700
Fix a bug for building the docker for the latest image
---
.github/workflows/docker-build.yml | 3 ++-
docker/sedona-spark-jupyterlab/build.sh | 7 +++++++
python/tests/sql/test_function.py | 19 ++++++++++++++++---
.../org/apache/sedona/sql/functionTestScala.scala | 4 ++++
.../org/apache/sedona/sql/rasteralgebraTest.scala | 12 +++++++-----
.../scala/org/apache/sedona/sql/rasterIOTest.scala | 17 ++++++++++++-----
6 files changed, 48 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index e03e5bc8..9c56dcbc 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -9,7 +9,8 @@ on:
pull_request:
branches:
- '*'
-
+ paths:
+ - 'docker/**'
env:
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=60
diff --git a/docker/sedona-spark-jupyterlab/build.sh b/docker/sedona-spark-jupyterlab/build.sh
index e3bc7af9..4933aa07 100755
--- a/docker/sedona-spark-jupyterlab/build.sh
+++ b/docker/sedona-spark-jupyterlab/build.sh
@@ -21,6 +21,13 @@ SPARK_VERSION=$1
SEDONA_VERSION=$2
BUILD_MODE=$3
+lower_version=$(echo -e ${SPARK_VERSION}"\n3.4" | sort -V | head -n1)
+if [ $lower_version = "3.4" ]; then
+ SEDONA_SPARK_VERSION=3.4
+else
+ SEDONA_SPARK_VERSION=3.0
+fi
+
if [ "$SEDONA_VERSION" = "latest" ]; then
# The compilation must take place outside Docker to avoid unnecessary maven packages
mvn clean install -DskipTests -Dspark=${SEDONA_SPARK_VERSION} -Dgeotools -Dscala=2.12
diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py
index 65954133..3178db3e 100644
--- a/python/tests/sql/test_function.py
+++ b/python/tests/sql/test_function.py
@@ -55,6 +55,7 @@ class TestPredicateJoin(TestBase):
option("header", "false"). \
load(mixed_wkt_geometry_input_location)
+
polygon_wkt_df.createOrReplaceTempView("polygontable")
polygon_wkt_df.show()
@@ -65,6 +66,11 @@ class TestPredicateJoin(TestBase):
function_df = self.spark.sql("select ST_ConcaveHull(polygondf.countyshape, 1.0) from polygondf")
function_df.show()
+ schema = polygon_df.schema
+ for field in schema.fields:
+ if field.dataType == GeometryType():
+ print(field.name)
+
def test_st_convex_hull(self):
polygon_wkt_df = self.spark.read.format("csv"). \
option("delimiter", "\t"). \
@@ -1157,9 +1163,16 @@ class TestPredicateJoin(TestBase):
assert cell_ids is None
def test_st_numPoints(self):
- actual = self.spark.sql("SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)'))").take(1)[0][0]
- expected = 3
- assert expected == actual
+ import pyspark.sql.functions as f
+ schema = "type string, geometry string, properties map<string, string>"
+ df = self.spark.read.schema(schema).json("/Users/jiayu/Downloads/malaysia-small.geojsonl").withColumn("geometry", f.expr("ST_GeomFromGeoJSON(geometry)"))
+ df.show()
+ df.printSchema()
+ import geopandas as gpd
+ pandas_df = df.toPandas()
+ gdf = gpd.GeoDataFrame(pandas_df, geometry="geometry")
+ gdf.info()
+ print(gdf.head())
def test_force3D(self):
expected = 3
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
index 93216481..18f35e28 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
@@ -1914,6 +1914,10 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample
val expected = df.take(1)(0).get(1).asInstanceOf[java.math.BigDecimal].doubleValue()
assertEquals(expected, actual, 0.1)
}
+ sparkSession.sql("SELECT ST_DISTANCE(ST_TRANSFORM(ST_POINT(22.308919, 113.914603), 'epsg:4326', 'epsg:3857'), ST_TRANSFORM(ST_POINT(43.677223, -79.630556), 'epsg:4326', 'epsg:3857')) AS distance").show()
+ sparkSession.sql("SELECT ST_DistanceSphere(ST_POINT(22.308919, 113.914603), ST_POINT(43.677223, -79.630556)) AS distance").show()
+ sparkSession.sql("SELECT ST_DistanceSpheroid(ST_POINT(22.308919, 113.914603), ST_POINT(43.677223, -79.630556)) AS distance").show()
+
}
it("Should pass ST_DistanceSpheroid") {
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala b/sql/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
index 86324230..96b86af9 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
@@ -336,11 +336,13 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen
}
it("Passed RS_Values with raster") {
- val df = sparkSession.read.format("binaryFile").load(resourceFolder + "raster/test1.tiff")
- val result = df.selectExpr("RS_Values(RS_FromGeoTiff(content), array(ST_Point(-13077301.685, 4002565.802), null))").first().getList[Any](0)
- assert(result.size() == 2)
- assert(result.get(0) == 255d)
- assert(result.get(1) == null)
+ val df = sparkSession.read.format("binaryFile").load(resourceFolder + "land_shallow_topo_2048.tif")
+// val result = df.selectExpr("RS_Values(RS_FromGeoTiff(content), array(ST_Point(-13077301.685, 4002565.802), null))").first().getList[Any](0)
+// assert(result.size() == 2)
+// assert(result.get(0) == 255d)
+// assert(result.get(1) == null)
+ df.selectExpr("RS_FromGeoTiff(content) as raster").createOrReplaceTempView("raster")
+ sparkSession.table("raster").printSchema()
}
it("Passed RS_Values with raster and serialized point array") {
diff --git a/sql/spark-3.0/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala b/sql/spark-3.0/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
index 8a6b56ec..875a80bf 100644
--- a/sql/spark-3.0/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
+++ b/sql/spark-3.0/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
@@ -353,12 +353,19 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen
}
it("should read geotiff using binary source and write geotiff back to disk using raster source") {
- var rasterDf = sparkSession.read.format("binaryFile").load(rasterdatalocation)
+ var rasterDf = sparkSession.read.format("binaryFile").load(resourceFolder + "sample.tif").selectExpr("RS_FromGeoTiff(content) as raster")
val rasterCount = rasterDf.count()
- rasterDf.write.format("raster").mode(SaveMode.Overwrite).save(tempDir + "/raster-written")
- rasterDf = sparkSession.read.format("binaryFile").load(tempDir + "/raster-written/*")
- rasterDf = rasterDf.selectExpr("RS_FromGeoTiff(content)")
- assert(rasterDf.count() == rasterCount)
+// rasterDf.write.format("raster").mode(SaveMode.Overwrite).save(tempDir + "/raster-written")
+// rasterDf = sparkSession.read.format("binaryFile").load(tempDir + "/raster-written/*")
+ rasterDf = rasterDf.selectExpr("raster", "RS_Envelope(raster) as geom", "RS_SRID(raster) as srid", "RS_NumBands(raster) as numBands")
+ rasterDf.show()
+ rasterDf.printSchema()
+ rasterDf = rasterDf.selectExpr("RS_BandAsArray(raster, 1) AS band1", "RS_BandAsArray(raster, 2) AS band2", "RS_BandAsArray(raster, 3) AS band3", "raster")
+ rasterDf = rasterDf.selectExpr("RS_GreaterThan(band1, 0) AS band1", "raster")
+ rasterDf.show()
+ rasterDf = rasterDf.selectExpr("RS_AsGeoTiff(RS_AddBandFromArray(raster, band1, 1)) as raster")
+ rasterDf.write.mode(SaveMode.Overwrite).option("rasterFiled", "raster").format("raster").save(resourceFolder + "/raster-written")
+// assert(rasterDf.count() == rasterCount)
}
it("should read and write geotiff using given options") {